99
1010import os
1111import time
12- from dataclasses import dataclass , field
12+ from dataclasses import dataclass , field , replace
1313
1414from gooddata_eval .core .agentic ._failed_runs import build_failed_runs
1515from gooddata_eval .core .agentic ._gate import (
3434from gooddata_eval .core .config import ReasoningEffort
3535from gooddata_eval .core .evaluators .visualization import (
3636 EvaluationResult ,
37+ ExpectationOptions ,
3738 _check_visualization_skill_activated ,
3839 _evaluate_against_candidates ,
3940 evaluation_result_detail ,
4041 with_execution ,
42+ with_options ,
4143)
4244from gooddata_eval .core .langfuse .item_scope import item_scope
4345from gooddata_eval .core .models import (
@@ -64,7 +66,8 @@ class RunResult:
6466 conversation_id : str
6567 actual_output : CreatedVisualization | None
6668 eval_result : EvaluationResult
67- best_expected : CreatedVisualization
69+ # None for an item that expects no chart.
70+ best_expected : CreatedVisualization | None
6871 total_turns : int
6972 total_steps : int
7073 reasoning_steps : list [str ] = field (default_factory = list )
@@ -76,6 +79,8 @@ class RunResult:
7679 exit_reason : LoopExit = LoopExit .BUDGET_EXHAUSTED
7780 # This run's own wall time, from just before its first chat call to the end of its scoring.
7881 run_latency_s : float = 0.0
82+ # Everything the final turn said, which the answer patterns are matched against.
83+ answer_text : str = ""
7984
8085
8186@dataclass
@@ -196,8 +201,13 @@ def _execute_single_run(
196201 expected_outputs : list [CreatedVisualization ],
197202 max_iterations : int = _DEFAULT_MAX_ITERATIONS ,
198203 requires_execution : bool = False ,
204+ options : ExpectationOptions | None = None ,
199205) -> RunResult :
200206 """Drive one full multi-turn conversation and evaluate the result."""
207+ options = _effective_options (options , requires_execution )
208+ one_turn = not options .expects_visualization or options .chart_optional
209+ if one_turn :
210+ max_iterations = min (max_iterations , 1 )
201211 total_turns = 0
202212 total_steps = 0
203213 all_tool_call_events : list [ToolCallEvent ] = []
@@ -207,7 +217,8 @@ def _execute_single_run(
207217 turn_offset = 0.0 # each turn's call_ts/ts restarts near 0 -- shift by prior turns' wall time
208218 reasoning_index_offset = 0 # ditto for ReasoningStepEvent.index, which also restarts per turn
209219 tool_index_offset = 0 # ditto for ToolCallEvent.index
210- simulated_response_guide = expected_outputs [0 ] # primary candidate guides the simulated user
220+ # The primary candidate guides the simulated user. An item with no chart never replies.
221+ simulated_response_guide = expected_outputs [0 ] if expected_outputs else None
211222
212223 # Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop that
213224 # simply runs out of range() is labelled correctly with no trailing else.
@@ -258,7 +269,11 @@ def _execute_single_run(
258269 if not response_text :
259270 exit_reason = LoopExit .AGENT_SILENT
260271 break
261- if iteration >= max_iterations - 1 :
272+ if one_turn :
273+ # Answering without a chart is an outcome this item accepts.
274+ exit_reason = LoopExit .SUCCESS
275+ break
276+ if iteration >= max_iterations - 1 or simulated_response_guide is None :
262277 break
263278
264279 try :
@@ -288,7 +303,11 @@ def _execute_single_run(
288303 actual_output = current_result .created_visualizations .objects [0 ]
289304
290305 eval_result , best_expected = _evaluate_against_candidates (expected_outputs , actual_output , skill_activated )
291- eval_result = with_execution (eval_result , all_tool_call_events , current_result .text_response , requires_execution )
306+ eval_result = with_execution (
307+ eval_result , all_tool_call_events , current_result .text_response , options .requires_execution
308+ )
309+ answer_text = render_answer_text (current_result )
310+ eval_result = with_options (eval_result , options , actual_output , all_tool_call_events , answer_text )
292311
293312 return RunResult (
294313 conversation_id = conversation_id ,
@@ -303,9 +322,16 @@ def _execute_single_run(
303322 reasoning_step_events = all_reasoning_step_events ,
304323 exit_reason = exit_reason ,
305324 run_latency_s = time .monotonic () - run_started ,
325+ answer_text = answer_text ,
306326 )
307327
308328
329+ def _effective_options (options : ExpectationOptions | None , requires_execution : bool ) -> ExpectationOptions :
330+ """``options`` with the separate ``requires_execution`` argument folded in."""
331+ options = options or ExpectationOptions ()
332+ return replace (options , requires_execution = True ) if requires_execution else options
333+
334+
309335def run_agentic_visualization (
310336 host : str ,
311337 token : str ,
@@ -319,6 +345,7 @@ def run_agentic_visualization(
319345 agent_id : str | None = None ,
320346 user_context : dict | None = None ,
321347 requires_execution : bool = False ,
348+ options : ExpectationOptions | None = None ,
322349) -> AgenticRunSummary :
323350 """Run K independent conversations and return evaluation results.
324351
@@ -328,7 +355,18 @@ def run_agentic_visualization(
328355 conversations created by this function are deleted on completion.
329356
330357 ``requires_execution`` fails a run whose agent built the chart but never ran it.
358+
359+ Raises:
360+ ValueError: ``expected_outputs`` and ``options.expects_visualization`` disagree. An
361+ empty candidate list is only right for an item that expects no chart, and the
362+ scorer would otherwise read the default ``expects_visualization`` and fail it.
331363 """
364+ expects_visualization = (options or ExpectationOptions ()).expects_visualization
365+ if expects_visualization != bool (expected_outputs ):
366+ raise ValueError (
367+ "expected_outputs must list the candidate charts unless options.expects_visualization is false, "
368+ f"got { len (expected_outputs )} candidate(s) with expects_visualization={ expects_visualization } "
369+ )
332370 client = ChatClient (
333371 host = host ,
334372 token = token ,
@@ -343,7 +381,9 @@ def run_agentic_visualization(
343381 conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client .create_conversation ()
344382 try :
345383 run_results .append (
346- _execute_single_run (client , conv_id_0 , question , expected_outputs , max_iterations , requires_execution )
384+ _execute_single_run (
385+ client , conv_id_0 , question , expected_outputs , max_iterations , requires_execution , options
386+ )
347387 )
348388 finally :
349389 if initial_conversation_id is None :
@@ -353,7 +393,9 @@ def run_agentic_visualization(
353393 conv_id = client .create_conversation ()
354394 try :
355395 run_results .append (
356- _execute_single_run (client , conv_id , question , expected_outputs , max_iterations , requires_execution )
396+ _execute_single_run (
397+ client , conv_id , question , expected_outputs , max_iterations , requires_execution , options
398+ )
357399 )
358400 finally :
359401 client .delete_conversation (conv_id )
@@ -393,6 +435,53 @@ def _run_detail(run: RunResult) -> dict:
393435 }
394436
395437
438+ def _strict_checks (ev : EvaluationResult ) -> dict [str , bool ]:
439+ """The scores one run is gated on, by their Langfuse names.
440+
441+ An item that expects no chart is scored on that and on its answer only: the chart checks
442+ would all read False and say nothing about it.
443+ """
444+ if not ev .expects_visualization or (ev .chart_optional and not ev .visualization_created ):
445+ checks = {"assertion-vis-no-chart" : not ev .visualization_created }
446+ else :
447+ checks = {
448+ "assertion-cross-ref-valid" : ev .cross_ref_valid ,
449+ "assertion-vis-metric" : ev .metrics_correct ,
450+ "assertion-vis-dimensions" : ev .dimensions_correct ,
451+ "assertion-vis-filters" : ev .filters_correct ,
452+ "assertion-vis-type" : ev .viz_type_hard ,
453+ }
454+ if ev .requires_execution :
455+ checks ["assertion-vis-executed" ] = ev .executed
456+ if ev .buckets_correct is not None :
457+ checks ["assertion-vis-buckets" ] = ev .buckets_correct
458+ if ev .format_correct is not None :
459+ checks ["assertion-vis-format" ] = ev .format_correct
460+ if ev .part_present is not None :
461+ checks ["assertion-vis-part-present" ] = ev .part_present
462+ if ev .answer_correct is not None :
463+ checks ["assertion-vis-answer" ] = ev .answer_correct
464+ return checks
465+
466+
467+ def _options_lines (ev : EvaluationResult , answer_text : str ) -> str :
468+ """The item-level verdicts for the failure summary; only the ones the item asks for."""
469+ lines = ""
470+ if not ev .expects_visualization or ev .chart_optional :
471+ lines += f" No Chart : { not ev .visualization_created } \n "
472+ if ev .answer_correct is not None :
473+ lines += f" Answer Correct : { ev .answer_correct } \n "
474+ lines += "" .join (f" { problem } \n " for problem in ev .answer_problems )
475+ if ev .answer_problems :
476+ lines += f" answer : { answer_text [:500 ]!r} \n "
477+ if ev .format_correct is not None :
478+ lines += f" Format Correct : { ev .format_correct } \n "
479+ lines += "" .join (f" { problem } \n " for problem in ev .format_problems )
480+ if ev .part_present is not None :
481+ lines += f" Part Present : { ev .part_present } \n "
482+ return lines
483+
484+
396485def _buckets_line (ev : EvaluationResult ) -> str :
397486 """The bucket-by-bucket verdict for the failure summary, or "" when the item does not ask for it."""
398487 lines = "" .join (f" { problem } \n " for problem in ev .metric_layout_problems )
@@ -437,6 +526,7 @@ def evaluate_agentic_visualization(
437526 submit_trace_link : SubmitTraceLink = run_trace_link_inline ,
438527 user_context : dict | None = None ,
439528 requires_execution : bool = False ,
529+ options : ExpectationOptions | None = None ,
440530 gate : EvalGate = DEFAULT_GATE ,
441531) -> AgenticEvalOutcome :
442532 """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure.
@@ -472,6 +562,7 @@ def evaluate_agentic_visualization(
472562 agent_id = agent_id ,
473563 user_context = user_context ,
474564 requires_execution = requires_execution ,
565+ options = options ,
475566 )
476567
477568 if langfuse is not None and dataset_item_id :
@@ -485,25 +576,12 @@ def _write_scores(ctx: RunTraceContext) -> None:
485576 for run_idx , run in enumerate (summary .run_results ):
486577 pt = ctx .trace (run .conversation_id )
487578 ev = run .eval_result
488- strict_checks = {
489- "assertion-cross-ref-valid" : ev .cross_ref_valid ,
490- "assertion-vis-metric" : ev .metrics_correct ,
491- "assertion-vis-dimensions" : ev .dimensions_correct ,
492- "assertion-vis-filters" : ev .filters_correct ,
493- "assertion-vis-type" : ev .viz_type_hard ,
494- }
495- if ev .requires_execution :
496- strict_checks ["assertion-vis-executed" ] = ev .executed
497- if ev .buckets_correct is not None :
498- strict_checks ["assertion-vis-buckets" ] = ev .buckets_correct
579+ strict_checks = _strict_checks (ev )
499580 with ctx .observe (pt , run_idx , conversation_id = run .conversation_id , output = strict_checks ) as tid :
500- ctx .score (tid , name = "assertion-cross-ref-valid" , value = ev .cross_ref_valid , data_type = "BOOLEAN" )
501- ctx .score (tid , name = "assertion-vis-metric" , value = ev .metrics_correct , data_type = "BOOLEAN" )
502- ctx .score (tid , name = "assertion-vis-dimensions" , value = ev .dimensions_correct , data_type = "BOOLEAN" )
503- ctx .score (tid , name = "assertion-vis-filters" , value = ev .filters_correct , data_type = "BOOLEAN" )
504- ctx .score (tid , name = "assertion-vis-type" , value = ev .viz_type_hard , data_type = "BOOLEAN" )
505- if ev .buckets_correct is not None :
506- ctx .score (tid , name = "assertion-vis-buckets" , value = ev .buckets_correct , data_type = "BOOLEAN" )
581+ for score_name , value in strict_checks .items ():
582+ # Scored below on every item, gating or not.
583+ if score_name != "assertion-vis-executed" :
584+ ctx .score (tid , name = score_name , value = value , data_type = "BOOLEAN" )
507585 ctx .score (tid , name = "skill_selection" , value = ev .skill_activated , data_type = "BOOLEAN" )
508586 # Scored on every item, gating or not, so the run rate is visible per model.
509587 ctx .score (tid , name = "assertion-vis-executed" , value = ev .executed , data_type = "BOOLEAN" )
@@ -572,7 +650,11 @@ def _write_scores(ctx: RunTraceContext) -> None:
572650 n = len (expected_outputs )
573651 candidate_note = f" (closest of { n } candidates)" if n > 1 else ""
574652 cross_ref_detail = (" → " + "; " .join (ev .cross_ref_errors )) if ev .cross_ref_errors else ""
575- expected_dump = best .best_expected .model_dump (exclude_none = True )
653+ expected_dump = (
654+ best .best_expected .model_dump (exclude_none = True )
655+ if best .best_expected is not None
656+ else {"expects_visualization" : False }
657+ )
576658 actual_dump = best .actual_output .model_dump (exclude_none = True ) if best .actual_output else None
577659 exc = VisualizationAssertionError (
578660 "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n "
@@ -604,6 +686,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
604686 f" Viz Type Hard : { ev .viz_type_hard } \n "
605687 f"{ _buckets_line (ev )} "
606688 f" Executed : { ev .executed } { ' (required)' if ev .requires_execution else '' } \n "
689+ f"{ _options_lines (ev , best .answer_text )} "
607690 f" Stated Value Matches : { ev .stated_value_matches } (reported only)\n "
608691 "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n "
609692 )
0 commit comments