2020 submit_trace_scoring ,
2121 utc_now ,
2222)
23+ from gooddata_eval .core .chat .render import render_answer_text
2324from gooddata_eval .core .chat .sse_client import ChatClient
2425from gooddata_eval .core .config import ReasoningEffort
26+ from gooddata_eval .core .evaluators ._maql import normalize_maql
2527from gooddata_eval .core .models import (
2628 AgenticAssertionError ,
2729 AgenticEvalOutcome ,
4042_DEFAULT_K = 1
4143_DEFAULT_MAX_ITERATIONS = 7
4244
43- _IFNULL_RE = re .compile (r"IFNULL\s*\([^,]+,\s*0\)" , re .IGNORECASE )
44- _SELECT_WRAP_RE = re .compile (r"^\s*\(\s*SELECT\s*\{([^}]+)\}\s*\)\s*$" , re .IGNORECASE )
45- _INNER_SELECT_RE = re .compile (r"\(\s*SELECT\s*\{([^}]+)\}\s*\)" , re .IGNORECASE )
46- # Matches whichever comes first: a {type/id} identifier reference or a quoted string
47- # literal -- both are case-sensitive data and must survive casefolding untouched.
48- # Everything else in MAQL (keywords, operators, numbers, punctuation) carries no
49- # case-sensitive meaning, per the MAQL reference (SELECT/BY/WHERE/FOR PREVIOUS/etc.
50- # are case-insensitive; only {..} identifiers and quoted literal values are not).
51- # Feeds _normalize_maql, the scoring comparator (_best_maql_match) -- do not widen this
52- # to handle \X escapes without confirming MAQL literals actually support backslash
53- # escaping (unconfirmed; see PR #1760 review). A wrong guess here silently changes
54- # maql_correct for the whole eval dataset, not just a hint. _no_where_clause_hint()
55- # below has its own, separately-scoped regex for that reason.
56- _PROTECTED_RE = re .compile (r"\{[^}]*\}|\"[^\"]*\"|'[^']*'" )
57-
58-
59- def _strip_outer_parens (s : str ) -> str :
60- """Strip one balanced layer of outer () if they wrap the entire expression."""
61- if not (s .startswith ("(" ) and s .endswith (")" )):
62- return s
63- depth = 0
64- for i , ch in enumerate (s ):
65- if ch == "(" :
66- depth += 1
67- elif ch == ")" :
68- depth -= 1
69- if depth == 0 and i < len (s ) - 1 :
70- return s # Closing paren found before end — not a simple outer wrapper
71- return s [1 :- 1 ].strip ()
72-
73-
74- def _casefold_outside_protected (s : str ) -> str :
75- """Lowercase MAQL keywords/operators while preserving case-sensitive {type/id}
76- identifiers and quoted string literal values (e.g. WHERE {label/x} = "Active")."""
77- parts = []
78- last = 0
79- for m in _PROTECTED_RE .finditer (s ):
80- parts .append (s [last : m .start ()].lower ())
81- parts .append (m .group (0 ))
82- last = m .end ()
83- parts .append (s [last :].lower ())
84- return "" .join (parts )
85-
86-
87- def _normalize_maql (maql : str ) -> str :
88- """Semantic normalisation: strip whitespace, unwrap IFNULL/SELECT wrappers, casefold keywords."""
89- if not maql :
90- return ""
91- m = maql .strip ()
92- m = _IFNULL_RE .sub (
93- lambda mo : _strip_outer_parens (mo .group (0 ).split ("," )[0 ].strip ()[len ("IFNULL(" ) :].strip ()),
94- m ,
95- )
96- m = _SELECT_WRAP_RE .sub (r"{\1}" , m )
97- m = _INNER_SELECT_RE .sub (r"{\1}" , m )
98- m = re .sub (r"\{\s+" , "{" , m )
99- m = re .sub (r"\s+\}" , "}" , m )
100- m = re .sub (r"\s+" , " " , m )
101- return _casefold_outside_protected (m .strip ())
102-
10345
10446def _best_maql_match (actual_maql : str , expected_outputs : list [dict ]) -> tuple [bool , str ]:
10547 """Try actual MAQL against every candidate; return (matched, best_expected_maql).
10648
10749 First match wins. First candidate is used for error reporting when none match.
10850 """
109- normalized_actual = _normalize_maql (actual_maql )
51+ normalized_actual = normalize_maql (actual_maql )
11052 for candidate in expected_outputs :
11153 expected_maql = candidate .get ("maql" , "" )
112- if normalized_actual == _normalize_maql (expected_maql ):
54+ if normalized_actual == normalize_maql (expected_maql ):
11355 return True , expected_maql
11456 return False , expected_outputs [0 ].get ("maql" , "" ) if expected_outputs else ""
11557
@@ -122,9 +64,9 @@ class SimulatedResponseError(RuntimeError):
12264 """
12365
12466
125- # Separate from _PROTECTED_RE on purpose: this one only feeds a same-turn LLM-prompt hint
126- # (see _no_where_clause_hint), never the scoring comparator, so it can afford to consume
127- # \X escape sequences inside quoted literals without risking maql_correct semantics.
67+ # Separate from evaluators._maql. _PROTECTED_RE on purpose: this one only feeds a same-turn
68+ # LLM-prompt hint (see _no_where_clause_hint), never the scoring comparator, so it can afford
69+ # to consume \X escape sequences inside quoted literals without risking maql_correct semantics.
12870_HINT_PROTECTED_RE = re .compile (r"\{[^}]*\}|\"(?:[^\"\\]|\\.)*\"|'(?:[^'\\]|\\.)*'" )
12971
13072
@@ -342,6 +284,8 @@ def _execute_single_metric_run(
342284 metric_result = candidate
343285 break
344286 response_text = (chat_result .text_response or "" ).strip ()
287+ if not response_text :
288+ response_text = render_answer_text (chat_result )
345289 if not response_text and not chat_result .tool_call_events :
346290 break
347291 if _iteration >= max_iterations - 1 :
0 commit comments