Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
209 changes: 209 additions & 0 deletions nodescraper/base/match_ignore.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
###############################################################################
#
# MIT License
#
# Copyright (c) 2026 Advanced Micro Devices, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
###############################################################################
import re
from dataclasses import dataclass
from typing import Optional, Sequence, Union

_MCE_BANK_RE = re.compile(r"\bMC(?P<bank>\d+)_STATUS\b", re.IGNORECASE)

MceBankSpec = Union[int, str]
IgnoreMatchRuleSpec = dict[str, object]


@dataclass(frozen=True)
class ParsedIgnoreMatchRule:
line_regex: Optional[re.Pattern[str]] = None
match_regex: Optional[re.Pattern[str]] = None
messages: Optional[frozenset[str]] = None
mce_banks: Optional[frozenset[int]] = None


def parse_mce_bank_spec(spec: Sequence[MceBankSpec]) -> frozenset[int]:
"""Expand MCA bank ids and inclusive ranges into a set of bank numbers.

Args:
spec: Bank ids, bank ranges like "60-63", or an empty sequence.

Returns:
frozenset[int]: MCA bank numbers.
"""
banks: set[int] = set()
for entry in spec:
if isinstance(entry, int):
if entry < 0:
raise ValueError(f"Invalid MCE bank number: {entry}")
banks.add(entry)
continue

token = str(entry).strip()
if not token:
raise ValueError("Empty MCE bank entry")

if "-" in token:
start_text, end_text = token.split("-", 1)
start = int(start_text.strip())
end = int(end_text.strip())
if start < 0 or end < 0 or start > end:
raise ValueError(f"Invalid MCE bank range: {entry}")
banks.update(range(start, end + 1))
continue

bank = int(token)
if bank < 0:
raise ValueError(f"Invalid MCE bank number: {entry}")
banks.add(bank)

return frozenset(banks)


def extract_mce_bank_from_line(line: str) -> Optional[int]:
"""Return the MCA bank number from a dmesg line, if present.

Args:
line: Single dmesg log line.

Returns:
Optional[int]: MCA bank number, or None when the line has no MCn_STATUS token.
"""
match = _MCE_BANK_RE.search(line)
if match is None:
return None
return int(match.group("bank"))


def extract_mce_banks_from_text(text: str) -> frozenset[int]:
"""Return all MCA bank numbers referenced in text.

Args:
text: Log line or regex match text.

Returns:
frozenset[int]: MCA bank numbers found in text.
"""
return frozenset(int(match.group("bank")) for match in _MCE_BANK_RE.finditer(text))


def parse_ignore_match_rules(
spec: Optional[Sequence[IgnoreMatchRuleSpec]],
) -> tuple[list[ParsedIgnoreMatchRule], frozenset[int]]:
"""Parse ignore_match_rules config into compiled skip rules and ignored MCA banks.

Args:
spec: Rule dicts using line_regex, match_regex, message, and/or mce_banks.

Returns:
tuple[list[ParsedIgnoreMatchRule], frozenset[int]]: Parsed rules and all ignored MCA banks.
"""
if not spec:
return [], frozenset()

parsed_rules: list[ParsedIgnoreMatchRule] = []
ignored_mce_banks: set[int] = set()

for index, raw_rule in enumerate(spec):
if not isinstance(raw_rule, dict):
raise ValueError(f"ignore_match_rules[{index}] must be a dict")

line_regex = raw_rule.get("line_regex")
match_regex = raw_rule.get("match_regex")
message = raw_rule.get("message")
mce_banks = raw_rule.get("mce_banks")

if line_regex is None and match_regex is None and mce_banks is None:
raise ValueError(
f"ignore_match_rules[{index}] must specify at least one of "
"line_regex, match_regex, or mce_banks"
)

messages: Optional[frozenset[str]] = None
if message is not None:
if isinstance(message, str):
messages = frozenset({message})
elif isinstance(message, list) and all(isinstance(item, str) for item in message):
messages = frozenset(message)
else:
raise ValueError(
f"ignore_match_rules[{index}].message must be a string or list of strings"
)

parsed_mce_banks: Optional[frozenset[int]] = None
if mce_banks is not None:
if not isinstance(mce_banks, list):
raise ValueError(f"ignore_match_rules[{index}].mce_banks must be a list")
parsed_mce_banks = parse_mce_bank_spec(mce_banks)
ignored_mce_banks.update(parsed_mce_banks)

parsed_rules.append(
ParsedIgnoreMatchRule(
line_regex=re.compile(str(line_regex)) if line_regex is not None else None,
match_regex=re.compile(str(match_regex)) if match_regex is not None else None,
messages=messages,
mce_banks=parsed_mce_banks,
)
)

return parsed_rules, frozenset(ignored_mce_banks)


def should_ignore_match(
*,
line: str,
match_text: str,
error_regex_message: str,
rules: Sequence[ParsedIgnoreMatchRule],
) -> bool:
"""Return True when any ignore rule matches the current regex hit.

Args:
line: Full log line containing the match.
match_text: Regex match text.
error_regex_message: ErrorRegex.message for the pattern that matched.
rules: Parsed ignore rules; first matching rule wins.

Returns:
bool: True when the match should be skipped.
"""
for rule in rules:
if rule.messages is not None and error_regex_message not in rule.messages:
continue

if rule.mce_banks is not None:
banks = extract_mce_banks_from_text(match_text)
if not banks:
line_bank = extract_mce_bank_from_line(line)
banks = frozenset({line_bank}) if line_bank is not None else frozenset()
if not banks or not banks.issubset(rule.mce_banks):
continue

if rule.line_regex is not None and rule.line_regex.search(line) is None:
continue

if rule.match_regex is not None and rule.match_regex.search(match_text) is None:
continue

return True

return False
63 changes: 62 additions & 1 deletion nodescraper/base/regexanalyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,11 @@
###############################################################################
import datetime
import re
from typing import Optional, Union
from typing import Optional, Sequence, Union

from pydantic import BaseModel

from nodescraper.base.match_ignore import ParsedIgnoreMatchRule, should_ignore_match
from nodescraper.enums import EventCategory, EventPriority
from nodescraper.generictypes import TAnalyzeArg, TDataModel
from nodescraper.interfaces.dataanalyzertask import DataAnalyzer
Expand Down Expand Up @@ -121,6 +122,51 @@ def _extract_timestamp_from_match_position(
timestamp_match = self.TIMESTAMP_PATTERN.search(first_line)
return timestamp_match.group(1) if timestamp_match else None

def _line_at_match_position(self, content: str, match_start: int) -> str:
"""Return the full line containing a regex match start position.

Args:
content: Full content being analyzed.
match_start: Start position of the regex match.

Returns:
str: Line text containing the match.
"""
line_start = content.rfind("\n", 0, match_start) + 1
line_end = content.find("\n", match_start)
if line_end == -1:
line_end = len(content)
return content[line_start:line_end]

def _should_ignore_regex_match(
self,
content: str,
match_start: int,
match_text: str,
error_regex_message: str,
ignore_match_rules: Sequence[ParsedIgnoreMatchRule],
) -> bool:
"""Return True when ignore_match_rules say to skip this regex hit.

Args:
content: Full content being analyzed.
match_start: Start position of the regex match.
match_text: Regex match text.
error_regex_message: ErrorRegex.message for the pattern that matched.
ignore_match_rules: Parsed ignore rules.

Returns:
bool: True when the match should be skipped.
"""
if not ignore_match_rules:
return False
return should_ignore_match(
line=self._line_at_match_position(content, match_start),
match_text=match_text,
error_regex_message=error_regex_message,
rules=ignore_match_rules,
)

def _convert_and_extend_error_regex(
self,
custom_regex: Optional[Union[list[ErrorRegex], list[dict]]],
Expand Down Expand Up @@ -198,13 +244,15 @@ def check_all_regexes(
group: bool = True,
num_timestamps: int = 3,
interval_to_collapse_event: int = 60,
ignore_match_rules: Optional[Sequence[ParsedIgnoreMatchRule]] = None,
) -> list[RegexEvent]:
"""Iterate over all ERROR_REGEX and check content for any matches

Enhanced with timestamp-based event collapsing:
- Extracts timestamps from matched lines
- Collapses events within interval_to_collapse_event seconds
- Prunes timestamp lists to keep first N and last N timestamps
- Skips matches that satisfy ignore_match_rules

Args:
content (str): content to match regex on
Expand All @@ -213,6 +261,7 @@ def check_all_regexes(
group (bool, optional): flag to control whether matches should be grouped together. Defaults to True.
num_timestamps (int, optional): maximum number of timestamps to keep for each event. Defaults to 3.
interval_to_collapse_event (int, optional): time interval in seconds to collapse events. Defaults to 60.
ignore_match_rules (Optional[Sequence[ParsedIgnoreMatchRule]], optional): Parsed skip rules. Defaults to None.

Returns:
list[RegexEvent]: list of regex event objects
Expand Down Expand Up @@ -246,8 +295,20 @@ def _is_within_interval(new_timestamp_str: str, existing_timestamps: list[str])
continue
return False

skip_rules = list(ignore_match_rules) if ignore_match_rules else []

for error_regex_obj in error_regex:
for match_obj in error_regex_obj.regex.finditer(content):
raw_match = match_obj.group(0)
if self._should_ignore_regex_match(
content,
match_obj.start(),
raw_match,
error_regex_obj.message,
skip_rules,
):
continue

# Extract timestamp from the line where match occurs
timestamp = self._extract_timestamp_from_match_position(content, match_obj.start())

Expand Down
11 changes: 6 additions & 5 deletions nodescraper/pluginregistry.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,11 +178,12 @@ def _valid_sub_class_check(in_cls: type, base_class: type) -> bool:
Returns:
bool: True if cls is a subclass of base_class, False otherwise.
"""
return (
inspect.isclass(in_cls)
and issubclass(in_cls, base_class)
and not inspect.isabstract(in_cls)
)
if not inspect.isclass(in_cls):
return False
try:
return issubclass(in_cls, base_class) and not inspect.isabstract(in_cls)
except TypeError:
return False

@staticmethod
def _load_connection_managers_uncached() -> dict[str, type]:
Expand Down
10 changes: 10 additions & 0 deletions nodescraper/plugins/inband/dmesg/analyzer_args.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@

from pydantic import Field

from nodescraper.base.match_ignore import IgnoreMatchRuleSpec
from nodescraper.base.regexanalyzer import ErrorRegex
from nodescraper.models import TimeRangeAnalysisArgs

Expand Down Expand Up @@ -69,3 +70,12 @@ class DmesgAnalyzerArgs(TimeRangeAnalysisArgs):
"(CPU, GPU BDF/block, etc.) reaches or exceeds this value."
),
)
ignore_match_rules: Optional[list[IgnoreMatchRuleSpec]] = Field(
default=None,
description=(
"Rules that skip regex matches during analysis. Each rule may use line_regex, "
"match_regex, message, and/or mce_banks. Within a rule all specified fields must "
"match; any matching rule suppresses the hit. mce_banks accepts bank ids and "
'inclusive ranges such as "60-63".'
),
)
Loading
Loading