Skip to content

Commit b7cb7d5

Browse files
TomkessTomkess
committed
Populate best_detail for every agentic test kind
Companion gap to the reasoning_steps fix above: run_agentic_items never copied any per-kind diagnostic info into ItemReport.best_detail, so every agentic result's "detail" field was always {} -- undiagnosable, unlike the single-shot path which gets this for free from ItemEvaluation.detail. AgenticEvalOutcome gains a `detail: dict` field, populated by each evaluate_agentic_* function from whatever its own Result/Evaluation class already tracks (no new capture needed) -- e.g. actual_maql for metric_skill, the full visualization per-check breakdown (now shared via evaluation_result_detail() between the single-shot and agentic paths so both report the identical shape), judge_reasoning/actual_output for the LLM-judge kinds (guardrail, general_question). Same *AssertionError idiom as reasoning_steps: attached as .detail on failure, returned on the AgenticEvalOutcome on success. cli/agentic_runner.py copies it onto item_report.best_detail in both branches.
1 parent 8dd6f12 commit b7cb7d5

20 files changed

Lines changed: 341 additions & 46 deletions

‎packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,8 @@ def _dispatch_agentic(
9090
"""Call the appropriate evaluate_agentic_* function for the item's test_kind.
9191
9292
Every evaluate_agentic_* function returns an AgenticEvalOutcome (reasoning_steps,
93-
conversation_id, response_id) on success and attaches the same three attributes to its
94-
raised *AssertionError on failure -- no kind is exempt.
93+
conversation_id, response_id, detail) on success and attaches the same four attributes
94+
to its raised *AssertionError on failure -- no kind is exempt.
9595
"""
9696
kind = item.test_kind
9797
eo = item.expected_output
@@ -241,19 +241,22 @@ def run_agentic_items(
241241
reasoning_steps = outcome.reasoning_steps
242242
conversation_id = outcome.conversation_id
243243
response_id = outcome.response_id
244+
detail = outcome.detail
244245
else:
245-
reasoning_steps, conversation_id, response_id = outcome, None, None
246+
reasoning_steps, conversation_id, response_id, detail = outcome, None, None, {}
246247
item_report.pass_at_k = True
247248
item_report.runs = k
248249
item_report.reasoning_steps = reasoning_steps or []
249250
item_report.conversation_id = conversation_id
250251
item_report.response_id = response_id
252+
item_report.best_detail = detail or {}
251253
except AssertionError as exc:
252254
item_report.pass_at_k = False
253255
item_report.runs = k
254256
item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or []
255257
item_report.conversation_id = getattr(exc, "conversation_id", None)
256258
item_report.response_id = getattr(exc, "response_id", None)
259+
item_report.best_detail = getattr(exc, "detail", None) or {}
257260
print(f"[agentic] {item.id} FAIL: {exc}", flush=True)
258261
except Exception as exc:
259262
item_report.error = f"{type(exc).__name__}: {exc}"

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -584,6 +584,7 @@ class AlertSkillAssertionError(AssertionError):
584584
reasoning_steps: list[str]
585585
conversation_id: str
586586
response_id: str | None
587+
detail: dict
587588

588589

589590
def evaluate_agentic_alert_skill(
@@ -697,9 +698,31 @@ def evaluate_agentic_alert_skill(
697698
exc.reasoning_steps = best.reasoning_steps
698699
exc.conversation_id = best.conversation_id
699700
exc.response_id = best.response_id
701+
exc.detail = {
702+
"alert_created": ev.alert_created,
703+
"operator_correct": ev.operator_correct,
704+
"threshold_correct": ev.threshold_correct,
705+
"trigger_correct": ev.trigger_correct,
706+
"filters_correct": ev.filters_correct,
707+
"metric_correct": ev.metric_correct,
708+
"recipients_correct": ev.recipients_correct,
709+
"actual_alert_arguments": best.actual_alert_arguments,
710+
}
700711
raise exc
712+
best = summary.best
713+
ev = best.eval
701714
return AgenticEvalOutcome(
702-
reasoning_steps=summary.best.reasoning_steps,
703-
conversation_id=summary.best.conversation_id,
704-
response_id=summary.best.response_id,
715+
reasoning_steps=best.reasoning_steps,
716+
conversation_id=best.conversation_id,
717+
response_id=best.response_id,
718+
detail={
719+
"alert_created": ev.alert_created,
720+
"operator_correct": ev.operator_correct,
721+
"threshold_correct": ev.threshold_correct,
722+
"trigger_correct": ev.trigger_correct,
723+
"filters_correct": ev.filters_correct,
724+
"metric_correct": ev.metric_correct,
725+
"recipients_correct": ev.recipients_correct,
726+
"actual_alert_arguments": best.actual_alert_arguments,
727+
},
705728
)

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -406,13 +406,32 @@ def run_agentic_conversation(
406406
)
407407

408408

409+
def _conversation_detail(result: ConversationResult) -> dict:
410+
return {
411+
"full_skill_coverage": result.full_skill_coverage,
412+
"total_clarification_turns": result.total_clarification_turns,
413+
"turns": [
414+
{
415+
"turn_id": tr.turn_id,
416+
"expected_skill": tr.expected_skill,
417+
"skill_routing": tr.skill_routing,
418+
"output_present": tr.output_present,
419+
"output_correct": tr.output_correct,
420+
"activated_skills": tr.activated_skills,
421+
}
422+
for tr in result.turn_results
423+
],
424+
}
425+
426+
409427
class ConversationAssertionError(AssertionError):
410428
"""Raised when a conversation evaluation fails."""
411429

412430
__tracebackhide__ = True
413431
reasoning_steps: list[str]
414432
conversation_id: str
415433
response_id: str | None
434+
detail: dict
416435

417436

418437
def evaluate_agentic_conversation(
@@ -524,9 +543,11 @@ def evaluate_agentic_conversation(
524543
exc.reasoning_steps = result.reasoning_steps
525544
exc.conversation_id = result.conversation_id
526545
exc.response_id = result.response_id
546+
exc.detail = _conversation_detail(result)
527547
raise exc
528548
return AgenticEvalOutcome(
529549
reasoning_steps=result.reasoning_steps,
530550
conversation_id=result.conversation_id,
531551
response_id=result.response_id,
552+
detail=_conversation_detail(result),
532553
)

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ class GeneralQuestionAssertionError(AssertionError):
152152
reasoning_steps: list[str]
153153
conversation_id: str
154154
response_id: str | None
155+
detail: dict
155156

156157

157158
def evaluate_agentic_general_question(
@@ -244,9 +245,20 @@ def evaluate_agentic_general_question(
244245
exc.reasoning_steps = best.reasoning_steps
245246
exc.conversation_id = best.conversation_id
246247
exc.response_id = best.response_id
248+
exc.detail = {
249+
"judge_passed": best.passed,
250+
"judge_reasoning": best.reasoning,
251+
"actual_output": best.actual_output,
252+
}
247253
raise exc
254+
best = summary.best
248255
return AgenticEvalOutcome(
249-
reasoning_steps=summary.best.reasoning_steps,
250-
conversation_id=summary.best.conversation_id,
251-
response_id=summary.best.response_id,
256+
reasoning_steps=best.reasoning_steps,
257+
conversation_id=best.conversation_id,
258+
response_id=best.response_id,
259+
detail={
260+
"judge_passed": best.passed,
261+
"judge_reasoning": best.reasoning,
262+
"actual_output": best.actual_output,
263+
},
252264
)

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ class GuardrailAssertionError(AssertionError):
149149
reasoning_steps: list[str]
150150
conversation_id: str
151151
response_id: str | None
152+
detail: dict
152153

153154

154155
def evaluate_agentic_guardrail(
@@ -240,9 +241,20 @@ def evaluate_agentic_guardrail(
240241
exc.reasoning_steps = best.reasoning_steps
241242
exc.conversation_id = best.conversation_id
242243
exc.response_id = best.response_id
244+
exc.detail = {
245+
"judge_passed": best.passed,
246+
"judge_reasoning": best.reasoning,
247+
"actual_output": best.actual_output,
248+
}
243249
raise exc
250+
best = summary.best
244251
return AgenticEvalOutcome(
245-
reasoning_steps=summary.best.reasoning_steps,
246-
conversation_id=summary.best.conversation_id,
247-
response_id=summary.best.response_id,
252+
reasoning_steps=best.reasoning_steps,
253+
conversation_id=best.conversation_id,
254+
response_id=best.response_id,
255+
detail={
256+
"judge_passed": best.passed,
257+
"judge_reasoning": best.reasoning,
258+
"actual_output": best.actual_output,
259+
},
248260
)

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,7 @@ class KdaSkillAssertionError(AssertionError):
328328
reasoning_steps: list[str]
329329
conversation_id: str
330330
response_id: str | None
331+
detail: dict
331332

332333

333334
def evaluate_agentic_kda_skill(
@@ -447,9 +448,29 @@ def evaluate_agentic_kda_skill(
447448
exc.reasoning_steps = best.reasoning_steps
448449
exc.conversation_id = best.conversation_id
449450
exc.response_id = best.response_id
451+
exc.detail = {
452+
"triggered": ev.triggered,
453+
"executed": ev.executed,
454+
"success": ev.success,
455+
"turn_completed": ev.turn_completed,
456+
"disambiguated": ev.disambiguated,
457+
"actual_create_args": best.actual_create_args,
458+
"actual_execute_result": best.actual_execute_result,
459+
}
450460
raise exc
461+
best = summary.best
462+
ev = best.evaluation
451463
return AgenticEvalOutcome(
452-
reasoning_steps=summary.best.reasoning_steps,
453-
conversation_id=summary.best.conversation_id,
454-
response_id=summary.best.response_id,
464+
reasoning_steps=best.reasoning_steps,
465+
conversation_id=best.conversation_id,
466+
response_id=best.response_id,
467+
detail={
468+
"triggered": ev.triggered,
469+
"executed": ev.executed,
470+
"success": ev.success,
471+
"turn_completed": ev.turn_completed,
472+
"disambiguated": ev.disambiguated,
473+
"actual_create_args": best.actual_create_args,
474+
"actual_execute_result": best.actual_execute_result,
475+
},
455476
)

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,7 @@ class MetricSkillAssertionError(AssertionError):
336336
reasoning_steps: list[str]
337337
conversation_id: str
338338
response_id: str | None
339+
detail: dict
339340

340341

341342
def evaluate_agentic_metric_skill(
@@ -438,9 +439,23 @@ def evaluate_agentic_metric_skill(
438439
exc.reasoning_steps = best.reasoning_steps
439440
exc.conversation_id = best.conversation_id
440441
exc.response_id = best.response_id
442+
exc.detail = {
443+
"metric_created": best.metric_created,
444+
"maql_correct": best.maql_correct,
445+
"expected_maql_candidates": [c.get("maql", "") for c in expected_outputs_list],
446+
"actual_maql": best.actual_maql,
447+
}
441448
raise exc
449+
best = summary.best
450+
expected_outputs_list = expected_output if isinstance(expected_output, list) else [expected_output]
442451
return AgenticEvalOutcome(
443-
reasoning_steps=summary.best.reasoning_steps,
444-
conversation_id=summary.best.conversation_id,
445-
response_id=summary.best.response_id,
452+
reasoning_steps=best.reasoning_steps,
453+
conversation_id=best.conversation_id,
454+
response_id=best.response_id,
455+
detail={
456+
"metric_created": best.metric_created,
457+
"maql_correct": best.maql_correct,
458+
"expected_maql_candidates": [c.get("maql", "") for c in expected_outputs_list],
459+
"actual_maql": best.actual_maql,
460+
},
446461
)

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ class SearchToolAssertionError(AssertionError):
142142
reasoning_steps: list[str]
143143
conversation_id: str
144144
response_id: str | None
145+
detail: dict
145146

146147

147148
def evaluate_agentic_search_tool(
@@ -236,9 +237,20 @@ def evaluate_agentic_search_tool(
236237
exc.reasoning_steps = best.reasoning_steps
237238
exc.conversation_id = best.conversation_id
238239
exc.response_id = best.response_id
240+
exc.detail = {
241+
"tool_selected": best.tool_selected,
242+
"tool_correct": best.tool_correct,
243+
"tool_call_names": best.tool_call_names,
244+
}
239245
raise exc
246+
best = summary.best
240247
return AgenticEvalOutcome(
241-
reasoning_steps=summary.best.reasoning_steps,
242-
conversation_id=summary.best.conversation_id,
243-
response_id=summary.best.response_id,
248+
reasoning_steps=best.reasoning_steps,
249+
conversation_id=best.conversation_id,
250+
response_id=best.response_id,
251+
detail={
252+
"tool_selected": best.tool_selected,
253+
"tool_correct": best.tool_correct,
254+
"tool_call_names": best.tool_call_names,
255+
},
244256
)

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
EvaluationResult,
1717
_check_visualization_skill_activated,
1818
_evaluate_against_candidates,
19+
evaluation_result_detail,
1920
)
2021
from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, ToolCallEvent
2122
from gooddata_eval.core.scoring import get_dimension_uri_set, get_metric_uri_set, uri_to_display_name
@@ -263,6 +264,7 @@ class VisualizationAssertionError(AssertionError):
263264
reasoning_steps: list[str]
264265
conversation_id: str
265266
response_id: str | None
267+
detail: dict
266268

267269

268270
def _filter_diff(category: str, ev: EvaluationResult) -> str:
@@ -432,9 +434,12 @@ def evaluate_agentic_visualization(
432434
exc.reasoning_steps = best.reasoning_steps
433435
exc.conversation_id = best.conversation_id
434436
exc.response_id = best.response_id
437+
exc.detail = evaluation_result_detail(ev)
435438
raise exc
439+
best = summary.best
436440
return AgenticEvalOutcome(
437-
reasoning_steps=summary.best.reasoning_steps,
438-
conversation_id=summary.best.conversation_id,
439-
response_id=summary.best.response_id,
441+
reasoning_steps=best.reasoning_steps,
442+
conversation_id=best.conversation_id,
443+
response_id=best.response_id,
444+
detail=evaluation_result_detail(best.eval_result),
440445
)

‎packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py‎

Lines changed: 28 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,33 @@ def _extract_actual(chat_result: ChatResult) -> CreatedVisualization | None:
159159
return cv.objects[0]
160160

161161

162+
def evaluation_result_detail(ev: EvaluationResult) -> dict:
163+
"""The per-check breakdown reported as ``detail`` for a visualization evaluation.
164+
165+
Shared by the single-shot evaluator below and the agentic path
166+
(``core/agentic/visualization.py``) so both report the exact same shape.
167+
"""
168+
return {
169+
"visualization_created": ev.visualization_created,
170+
"cross_ref_valid": ev.cross_ref_valid,
171+
"cross_ref_errors": ev.cross_ref_errors,
172+
"metrics_correct": ev.metrics_correct,
173+
"dimensions_correct": ev.dimensions_correct,
174+
"filters_correct": ev.filters_correct,
175+
"filter_date_score": ev.filter_date_score,
176+
"filter_ranking_score": ev.filter_ranking_score,
177+
"filter_attribute_score": ev.filter_attribute_score,
178+
"viz_type_hard": ev.viz_type_hard,
179+
"skill_activated": ev.skill_activated,
180+
"expected_metric_uris": sorted(ev.expected_metric_uris),
181+
"actual_metric_uris": sorted(ev.actual_metric_uris),
182+
"expected_dim_uris": sorted(ev.expected_dim_uris),
183+
"actual_dim_uris": sorted(ev.actual_dim_uris),
184+
"expected_filters": ev.expected_filters,
185+
"actual_filters": ev.actual_filters,
186+
}
187+
188+
162189
class VisualizationEvaluator:
163190
test_kind = "visualization"
164191

@@ -170,23 +197,5 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation
170197
return ItemEvaluation(
171198
passed=ev.strict_pass,
172199
rank_key=(ev.strict_pass, ev.strict_checks_passed_count),
173-
detail={
174-
"visualization_created": ev.visualization_created,
175-
"cross_ref_valid": ev.cross_ref_valid,
176-
"cross_ref_errors": ev.cross_ref_errors,
177-
"metrics_correct": ev.metrics_correct,
178-
"dimensions_correct": ev.dimensions_correct,
179-
"filters_correct": ev.filters_correct,
180-
"filter_date_score": ev.filter_date_score,
181-
"filter_ranking_score": ev.filter_ranking_score,
182-
"filter_attribute_score": ev.filter_attribute_score,
183-
"viz_type_hard": ev.viz_type_hard,
184-
"skill_activated": ev.skill_activated,
185-
"expected_metric_uris": sorted(ev.expected_metric_uris),
186-
"actual_metric_uris": sorted(ev.actual_metric_uris),
187-
"expected_dim_uris": sorted(ev.expected_dim_uris),
188-
"actual_dim_uris": sorted(ev.actual_dim_uris),
189-
"expected_filters": ev.expected_filters,
190-
"actual_filters": ev.actual_filters,
191-
},
200+
detail=evaluation_result_detail(ev),
192201
)

0 commit comments

Comments
 (0)