Skip to content

Commit 7642642

Browse files
huyenthanh09claude
andcommitted
test(gooddata-eval): check answers without a chart, number formats and the chart part
A visualization expectation can now state, beside its candidate charts: - `expects_visualization: false`: the agent must answer without a chart, for example by asking for a part the request leaves out. The run stops after one turn and publishes `assertion-vis-no-chart`. - `chart_optional: true`: either an answer without a chart or a chart that matches a candidate is right, for a request the chart type cannot draw as asked. The run stops after one turn. - `answer_must_include` / `answer_must_not_match`: regular expressions matched without regard to case against the final answer, published as `assertion-vis-answer`. A typographic apostrophe reads as a plain one. - `format_intents`: how a metric's numbers must read. The agent's own `format_example` from the chart tool call is read back against the intent and compared as a number, so `#,##0.0M` showing 4,221,301,265.5M for "in millions" fails (GDAI-2592). Published as `assertion-vis-format`. - `require_visualization_part`: the chart must come back as a part of the answer, not only as tool arguments. Published as `assertion-vis-part-present`. `expectation_options()` reads and validates these keys before any request. Every key is opt-in, so existing items score as before. A run keeps the final answer text, and the failure summary shows it when an answer check fails. JIRA: QA-29722 risk: nonprod Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 309b202 commit 7642642

8 files changed

Lines changed: 1105 additions & 38 deletions

File tree

‎packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
from gooddata_eval.core.agentic.what_if import evaluate_agentic_what_if
2727
from gooddata_eval.core.config import ReasoningEffort
2828
from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError
29-
from gooddata_eval.core.evaluators.visualization import requires_execution_of
29+
from gooddata_eval.core.evaluators.visualization import expectation_options
3030
from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem
3131
from gooddata_eval.core.runner import EvalReport, ItemReport
3232
from gooddata_eval.core.timing import TraceRecorder
@@ -119,6 +119,8 @@ def _parse_visualization_expected(expected_output: Any) -> list[CreatedVisualiza
119119
{"visualization": {...}} or {"visualization": [{...}]} <- single/multi candidate
120120
[{"visualization": {...}}, ...] <- bare list
121121
"""
122+
if isinstance(expected_output, dict) and expected_output.get("expects_visualization") is False:
123+
return []
122124
if isinstance(expected_output, dict):
123125
raw_list = expected_output.get("expected_outputs")
124126
if raw_list is not None:
@@ -187,7 +189,7 @@ def _dispatch_agentic(
187189
workspace_id=workspace_id,
188190
question=item.question,
189191
expected_outputs=_parse_visualization_expected(eo),
190-
requires_execution=requires_execution_of(eo),
192+
options=expectation_options(eo),
191193
k=k,
192194
gate=gate,
193195
agent_id=agent_id,

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py‎

Lines changed: 109 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
import os
1111
import time
12-
from dataclasses import dataclass, field
12+
from dataclasses import dataclass, field, replace
1313

1414
from gooddata_eval.core.agentic._failed_runs import build_failed_runs
1515
from gooddata_eval.core.agentic._gate import (
@@ -34,10 +34,12 @@
3434
from gooddata_eval.core.config import ReasoningEffort
3535
from gooddata_eval.core.evaluators.visualization import (
3636
EvaluationResult,
37+
ExpectationOptions,
3738
_check_visualization_skill_activated,
3839
_evaluate_against_candidates,
3940
evaluation_result_detail,
4041
with_execution,
42+
with_options,
4143
)
4244
from gooddata_eval.core.langfuse.item_scope import item_scope
4345
from gooddata_eval.core.models import (
@@ -64,7 +66,8 @@ class RunResult:
6466
conversation_id: str
6567
actual_output: CreatedVisualization | None
6668
eval_result: EvaluationResult
67-
best_expected: CreatedVisualization
69+
# None for an item that expects no chart.
70+
best_expected: CreatedVisualization | None
6871
total_turns: int
6972
total_steps: int
7073
reasoning_steps: list[str] = field(default_factory=list)
@@ -76,6 +79,8 @@ class RunResult:
7679
exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED
7780
# This run's own wall time, from just before its first chat call to the end of its scoring.
7881
run_latency_s: float = 0.0
82+
# Everything the final turn said, which the answer patterns are matched against.
83+
answer_text: str = ""
7984

8085

8186
@dataclass
@@ -196,8 +201,13 @@ def _execute_single_run(
196201
expected_outputs: list[CreatedVisualization],
197202
max_iterations: int = _DEFAULT_MAX_ITERATIONS,
198203
requires_execution: bool = False,
204+
options: ExpectationOptions | None = None,
199205
) -> RunResult:
200206
"""Drive one full multi-turn conversation and evaluate the result."""
207+
options = _effective_options(options, requires_execution)
208+
one_turn = not options.expects_visualization or options.chart_optional
209+
if one_turn:
210+
max_iterations = min(max_iterations, 1)
201211
total_turns = 0
202212
total_steps = 0
203213
all_tool_call_events: list[ToolCallEvent] = []
@@ -207,7 +217,8 @@ def _execute_single_run(
207217
turn_offset = 0.0 # each turn's call_ts/ts restarts near 0 -- shift by prior turns' wall time
208218
reasoning_index_offset = 0 # ditto for ReasoningStepEvent.index, which also restarts per turn
209219
tool_index_offset = 0 # ditto for ToolCallEvent.index
210-
simulated_response_guide = expected_outputs[0] # primary candidate guides the simulated user
220+
# The primary candidate guides the simulated user. An item with no chart never replies.
221+
simulated_response_guide = expected_outputs[0] if expected_outputs else None
211222

212223
# Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop that
213224
# simply runs out of range() is labelled correctly with no trailing else.
@@ -258,7 +269,11 @@ def _execute_single_run(
258269
if not response_text:
259270
exit_reason = LoopExit.AGENT_SILENT
260271
break
261-
if iteration >= max_iterations - 1:
272+
if one_turn:
273+
# Answering without a chart is an outcome this item accepts.
274+
exit_reason = LoopExit.SUCCESS
275+
break
276+
if iteration >= max_iterations - 1 or simulated_response_guide is None:
262277
break
263278

264279
try:
@@ -288,7 +303,11 @@ def _execute_single_run(
288303
actual_output = current_result.created_visualizations.objects[0]
289304

290305
eval_result, best_expected = _evaluate_against_candidates(expected_outputs, actual_output, skill_activated)
291-
eval_result = with_execution(eval_result, all_tool_call_events, current_result.text_response, requires_execution)
306+
eval_result = with_execution(
307+
eval_result, all_tool_call_events, current_result.text_response, options.requires_execution
308+
)
309+
answer_text = render_answer_text(current_result)
310+
eval_result = with_options(eval_result, options, actual_output, all_tool_call_events, answer_text)
292311

293312
return RunResult(
294313
conversation_id=conversation_id,
@@ -303,9 +322,16 @@ def _execute_single_run(
303322
reasoning_step_events=all_reasoning_step_events,
304323
exit_reason=exit_reason,
305324
run_latency_s=time.monotonic() - run_started,
325+
answer_text=answer_text,
306326
)
307327

308328

329+
def _effective_options(options: ExpectationOptions | None, requires_execution: bool) -> ExpectationOptions:
330+
"""``options`` with the separate ``requires_execution`` argument folded in."""
331+
options = options or ExpectationOptions()
332+
return replace(options, requires_execution=True) if requires_execution else options
333+
334+
309335
def run_agentic_visualization(
310336
host: str,
311337
token: str,
@@ -319,6 +345,7 @@ def run_agentic_visualization(
319345
agent_id: str | None = None,
320346
user_context: dict | None = None,
321347
requires_execution: bool = False,
348+
options: ExpectationOptions | None = None,
322349
) -> AgenticRunSummary:
323350
"""Run K independent conversations and return evaluation results.
324351
@@ -328,7 +355,18 @@ def run_agentic_visualization(
328355
conversations created by this function are deleted on completion.
329356
330357
``requires_execution`` fails a run whose agent built the chart but never ran it.
358+
359+
Raises:
360+
ValueError: ``expected_outputs`` and ``options.expects_visualization`` disagree. An
361+
empty candidate list is only right for an item that expects no chart, and the
362+
scorer would otherwise read the default ``expects_visualization`` and fail it.
331363
"""
364+
expects_visualization = (options or ExpectationOptions()).expects_visualization
365+
if expects_visualization != bool(expected_outputs):
366+
raise ValueError(
367+
"expected_outputs must list the candidate charts unless options.expects_visualization is false, "
368+
f"got {len(expected_outputs)} candidate(s) with expects_visualization={expects_visualization}"
369+
)
332370
client = ChatClient(
333371
host=host,
334372
token=token,
@@ -343,7 +381,9 @@ def run_agentic_visualization(
343381
conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation()
344382
try:
345383
run_results.append(
346-
_execute_single_run(client, conv_id_0, question, expected_outputs, max_iterations, requires_execution)
384+
_execute_single_run(
385+
client, conv_id_0, question, expected_outputs, max_iterations, requires_execution, options
386+
)
347387
)
348388
finally:
349389
if initial_conversation_id is None:
@@ -353,7 +393,9 @@ def run_agentic_visualization(
353393
conv_id = client.create_conversation()
354394
try:
355395
run_results.append(
356-
_execute_single_run(client, conv_id, question, expected_outputs, max_iterations, requires_execution)
396+
_execute_single_run(
397+
client, conv_id, question, expected_outputs, max_iterations, requires_execution, options
398+
)
357399
)
358400
finally:
359401
client.delete_conversation(conv_id)
@@ -393,6 +435,53 @@ def _run_detail(run: RunResult) -> dict:
393435
}
394436

395437

438+
def _strict_checks(ev: EvaluationResult) -> dict[str, bool]:
439+
"""The scores one run is gated on, by their Langfuse names.
440+
441+
An item that expects no chart is scored on that and on its answer only: the chart checks
442+
would all read False and say nothing about it.
443+
"""
444+
if not ev.expects_visualization or (ev.chart_optional and not ev.visualization_created):
445+
checks = {"assertion-vis-no-chart": not ev.visualization_created}
446+
else:
447+
checks = {
448+
"assertion-cross-ref-valid": ev.cross_ref_valid,
449+
"assertion-vis-metric": ev.metrics_correct,
450+
"assertion-vis-dimensions": ev.dimensions_correct,
451+
"assertion-vis-filters": ev.filters_correct,
452+
"assertion-vis-type": ev.viz_type_hard,
453+
}
454+
if ev.requires_execution:
455+
checks["assertion-vis-executed"] = ev.executed
456+
if ev.buckets_correct is not None:
457+
checks["assertion-vis-buckets"] = ev.buckets_correct
458+
if ev.format_correct is not None:
459+
checks["assertion-vis-format"] = ev.format_correct
460+
if ev.part_present is not None:
461+
checks["assertion-vis-part-present"] = ev.part_present
462+
if ev.answer_correct is not None:
463+
checks["assertion-vis-answer"] = ev.answer_correct
464+
return checks
465+
466+
467+
def _options_lines(ev: EvaluationResult, answer_text: str) -> str:
468+
"""The item-level verdicts for the failure summary; only the ones the item asks for."""
469+
lines = ""
470+
if not ev.expects_visualization or ev.chart_optional:
471+
lines += f" No Chart : {not ev.visualization_created}\n"
472+
if ev.answer_correct is not None:
473+
lines += f" Answer Correct : {ev.answer_correct}\n"
474+
lines += "".join(f" {problem}\n" for problem in ev.answer_problems)
475+
if ev.answer_problems:
476+
lines += f" answer : {answer_text[:500]!r}\n"
477+
if ev.format_correct is not None:
478+
lines += f" Format Correct : {ev.format_correct}\n"
479+
lines += "".join(f" {problem}\n" for problem in ev.format_problems)
480+
if ev.part_present is not None:
481+
lines += f" Part Present : {ev.part_present}\n"
482+
return lines
483+
484+
396485
def _buckets_line(ev: EvaluationResult) -> str:
397486
"""The bucket-by-bucket verdict for the failure summary, or "" when the item does not ask for it."""
398487
lines = "".join(f" {problem}\n" for problem in ev.metric_layout_problems)
@@ -437,6 +526,7 @@ def evaluate_agentic_visualization(
437526
submit_trace_link: SubmitTraceLink = run_trace_link_inline,
438527
user_context: dict | None = None,
439528
requires_execution: bool = False,
529+
options: ExpectationOptions | None = None,
440530
gate: EvalGate = DEFAULT_GATE,
441531
) -> AgenticEvalOutcome:
442532
"""Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure.
@@ -472,6 +562,7 @@ def evaluate_agentic_visualization(
472562
agent_id=agent_id,
473563
user_context=user_context,
474564
requires_execution=requires_execution,
565+
options=options,
475566
)
476567

477568
if langfuse is not None and dataset_item_id:
@@ -485,25 +576,12 @@ def _write_scores(ctx: RunTraceContext) -> None:
485576
for run_idx, run in enumerate(summary.run_results):
486577
pt = ctx.trace(run.conversation_id)
487578
ev = run.eval_result
488-
strict_checks = {
489-
"assertion-cross-ref-valid": ev.cross_ref_valid,
490-
"assertion-vis-metric": ev.metrics_correct,
491-
"assertion-vis-dimensions": ev.dimensions_correct,
492-
"assertion-vis-filters": ev.filters_correct,
493-
"assertion-vis-type": ev.viz_type_hard,
494-
}
495-
if ev.requires_execution:
496-
strict_checks["assertion-vis-executed"] = ev.executed
497-
if ev.buckets_correct is not None:
498-
strict_checks["assertion-vis-buckets"] = ev.buckets_correct
579+
strict_checks = _strict_checks(ev)
499580
with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid:
500-
ctx.score(tid, name="assertion-cross-ref-valid", value=ev.cross_ref_valid, data_type="BOOLEAN")
501-
ctx.score(tid, name="assertion-vis-metric", value=ev.metrics_correct, data_type="BOOLEAN")
502-
ctx.score(tid, name="assertion-vis-dimensions", value=ev.dimensions_correct, data_type="BOOLEAN")
503-
ctx.score(tid, name="assertion-vis-filters", value=ev.filters_correct, data_type="BOOLEAN")
504-
ctx.score(tid, name="assertion-vis-type", value=ev.viz_type_hard, data_type="BOOLEAN")
505-
if ev.buckets_correct is not None:
506-
ctx.score(tid, name="assertion-vis-buckets", value=ev.buckets_correct, data_type="BOOLEAN")
581+
for score_name, value in strict_checks.items():
582+
# Scored below on every item, gating or not.
583+
if score_name != "assertion-vis-executed":
584+
ctx.score(tid, name=score_name, value=value, data_type="BOOLEAN")
507585
ctx.score(tid, name="skill_selection", value=ev.skill_activated, data_type="BOOLEAN")
508586
# Scored on every item, gating or not, so the run rate is visible per model.
509587
ctx.score(tid, name="assertion-vis-executed", value=ev.executed, data_type="BOOLEAN")
@@ -572,7 +650,11 @@ def _write_scores(ctx: RunTraceContext) -> None:
572650
n = len(expected_outputs)
573651
candidate_note = f" (closest of {n} candidates)" if n > 1 else ""
574652
cross_ref_detail = (" → " + "; ".join(ev.cross_ref_errors)) if ev.cross_ref_errors else ""
575-
expected_dump = best.best_expected.model_dump(exclude_none=True)
653+
expected_dump = (
654+
best.best_expected.model_dump(exclude_none=True)
655+
if best.best_expected is not None
656+
else {"expects_visualization": False}
657+
)
576658
actual_dump = best.actual_output.model_dump(exclude_none=True) if best.actual_output else None
577659
exc = VisualizationAssertionError(
578660
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"
@@ -604,6 +686,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
604686
f" Viz Type Hard : {ev.viz_type_hard}\n"
605687
f"{_buckets_line(ev)}"
606688
f" Executed : {ev.executed}{' (required)' if ev.requires_execution else ''}\n"
689+
f"{_options_lines(ev, best.answer_text)}"
607690
f" Stated Value Matches : {ev.stated_value_matches} (reported only)\n"
608691
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"
609692
)

0 commit comments

Comments
 (0)