From d12931e0e7ca807709642611163e564944ae005a Mon Sep 17 00:00:00 2001 From: Frank Huynh Date: Thu, 24 Sep 2026 11:06:10 +0700 Subject: [PATCH] feat(gooddata-eval): evaluate the agentic dashboard-creation skill Adds the agentic_dashboard_skill kind: it drives the conversation until the agent produces a draft, then scores that draft. The gate holds only what the agent decided -- the draft call succeeded, the response carries the expected part, every expected chart is present, every tab's date filter matches, and it authored at least the required number of charts. Three observations sit beside the gate and never fail a run: whether a set_skills call activated dashboard_builder, whether the response's references carried every widget id, and whether the widget titles matched. They are scored rather than only printed, because each of them is a decision to stop failing on something, and without a score nothing would record how often it happens. The references one matters most: gen-ai rejects a draft naming an unresolvable visualization before it can succeed, so a widget missing from the references means reference building degraded on the way out, which is a platform fault and must not be scored as the model's. A chart the fixture gives an id is matched on that id alone, since the widget title is title_override or fallback_title and the override is the model's own choice. A chart the fixture marks with a null id has no identity to match on, so there the title is the match, taken among the authored charts only -- matching it against every chart would let an existing one of the same title satisfy it. The simulated user is deterministic rather than an LLM call: when the agent asks back instead of drafting, the only things it still needs are the charts and the date range, and both come straight from the expectation. That reply is byte-identical every turn, so the loop sends it once and stops. Repeating it answers nothing, and when the agent asked something the expectation does not cover, repeating it hides the signal that the question was too vague. Two turns also keep the worst case inside the per-test timeout gdc-nas derives. The expectation is validated before the first request: a fixture with no visualizations would pass every chart check vacuously, and a date range the simulated user cannot phrase would otherwise surface only on the branch where the agent asks back, passing or crashing depending on what the model chose. jira: QA-29347 risk: low Co-Authored-By: Claude Opus 5 (1M context) --- .../src/gooddata_eval/cli/agentic_runner.py | 18 + .../gooddata_eval/core/agentic/__init__.py | 14 + .../core/agentic/dashboard_skill.py | 742 ++++++++++++++++++ .../tests/test_agentic_dashboard_skill.py | 679 ++++++++++++++++ .../tests/test_agentic_runner.py | 5 + .../gooddata-eval/tests/test_trace_linker.py | 1 + 6 files changed, 1459 insertions(+) create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py create mode 100644 packages/gooddata-eval/tests/test_agentic_dashboard_skill.py diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 129f4c5cf..3e0a2ef50 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -13,6 +13,7 @@ from gooddata_eval.core.agentic._trace_linker import BackgroundTraceLinker, SubmitTraceLink, run_trace_link_inline from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation +from gooddata_eval.core.agentic.dashboard_skill import evaluate_agentic_dashboard_skill from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill @@ -40,6 +41,7 @@ class _LfKw(TypedDict, total=False): "agentic_visualization", # experimental: expected_output.expected_outputs (multi-candidate) "agentic_metric_skill", "agentic_alert_skill", + "agentic_dashboard_skill", "agentic_search", "agentic_general_question", "agentic_guardrail", @@ -85,6 +87,10 @@ class _LfKw(TypedDict, total=False): # create_key_driver_analysis with no cleanup, and while the evaluator only ever reads that # call's ARGUMENTS -- never a created object id -- whether the platform persists anything is # unverified. Move it to the allowlist once someone confirms it does not. +# +# agentic_dashboard_skill is absent by default rather than by evidence: gen-ai holds the draft and +# any chart it authors in conversation state and writes neither until a user saves from the UI, so +# it is a candidate for the allowlist once the dataset has runs behind it. WORKSPACE_MUTATING_TEST_KINDS = frozenset(AGENTIC_TEST_KINDS) - PARALLEL_SAFE_TEST_KINDS @@ -183,6 +189,18 @@ def _dispatch_agentic( agent_id=agent_id, **lf_kw, ) + elif kind == "agentic_dashboard_skill": + return evaluate_agentic_dashboard_skill( + host=host, + token=token, + workspace_id=workspace_id, + question=item.question, + expected_output=eo if isinstance(eo, dict) else {}, + k=k, + gate=gate, + agent_id=agent_id, + **lf_kw, + ) elif kind == "agentic_alert_skill": return evaluate_agentic_alert_skill( host=host, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py index 89e93dde8..3d74a4b9f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py @@ -16,6 +16,14 @@ evaluate_agentic_conversation, run_agentic_conversation, ) +from gooddata_eval.core.agentic.dashboard_skill import ( + AgenticDashboardSummary, + DashboardEvaluation, + DashboardRunResult, + DashboardSkillAssertionError, + evaluate_agentic_dashboard_skill, + run_agentic_dashboard_skill, +) from gooddata_eval.core.agentic.general_question import ( AgenticGeneralQuestionSummary, GeneralQuestionAssertionError, @@ -62,6 +70,7 @@ __all__ = [ "AgenticAlertSummary", + "AgenticDashboardSummary", "AgenticGeneralQuestionSummary", "AgenticGuardrailSummary", "AgenticKdaSummary", @@ -74,6 +83,9 @@ "ConversationAssertionError", "ConversationFixture", "ConversationResult", + "DashboardEvaluation", + "DashboardRunResult", + "DashboardSkillAssertionError", "GeneralQuestionAssertionError", "GeneralQuestionResult", "GuardrailAssertionError", @@ -91,6 +103,7 @@ "VisualizationAssertionError", "evaluate_agentic_alert_skill", "evaluate_agentic_conversation", + "evaluate_agentic_dashboard_skill", "evaluate_agentic_general_question", "evaluate_agentic_guardrail", "evaluate_agentic_kda_skill", @@ -99,6 +112,7 @@ "evaluate_agentic_visualization", "run_agentic_alert_skill", "run_agentic_conversation", + "run_agentic_dashboard_skill", "run_agentic_general_question", "run_agentic_guardrail", "run_agentic_kda_skill", diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py new file mode 100644 index 000000000..81eeabe4f --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py @@ -0,0 +1,742 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Agentic dashboard-skill evaluation runner.""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any + +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) +from gooddata_eval.core.chat.render import render_answer_text +from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ChatResult, + ReasoningStepEvent, + ToolCallEvent, + build_latency_breakdown, + shift_and_index_events, +) +from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings + +_DEFAULT_K = 1 +# Matches kda_skill and visualization. The reply this loop sends is built from the expectation +# and is byte-identical every turn, so the extra rounds are not there to say anything new -- +# they are slack for a turn that produced nothing. A turn can come back empty or answer without +# drafting for reasons that are not the model's, and with a tight cap that one wasted turn +# spends the only reply the case had; the run then fails as "no draft" when nothing was wrong +# with the agent. Fewer than metric_skill's seven because those rounds do carry new content: +# its reply is generated per turn by an LLM. +# +# The cost of the slack is that four turns at ChatClient's 300s read timeout exceed the 720s +# per-test timeout gdc-nas derives for a k=1 dataset, so a run that stalls on every turn is cut +# short by pytest rather than by this loop. Measured turns are nowhere near that -- a two-turn +# case completes in about four minutes -- so the ceiling only binds when something is already +# badly wrong. +_DEFAULT_MAX_ITERATIONS = 4 + +_DRAFT_TOOL = "draft_dashboard" +_SET_SKILLS_TOOL = "set_skills" +_BUILDER_SKILL = "dashboard_builder" + + +def _norm(text: str) -> str: + """Case- and whitespace-insensitive form used for every title comparison.""" + return " ".join(text.split()).casefold() + + +def _extract_draft_result(tool_call_events: list[ToolCallEvent]) -> dict | None: + """Result payload of the ``draft_dashboard`` call that produced a draft. + + Takes the most recent *successful* call: when the agent retries after a rejected + draft, the earlier failed attempt must not shadow the one that worked. + """ + for tc in reversed(tool_call_events): + if tc.function_name != _DRAFT_TOOL or not tc.result: + continue + result_data = tc.parsed_result() + if not isinstance(result_data, dict): + continue + payload = result_data.get("data", result_data) + if not isinstance(payload, dict) or payload.get("status") != "success": + continue + return payload + return None + + +def _builder_skill_activated(tool_call_events: list[ToolCallEvent]) -> bool: + """Whether any ``set_skills`` result activated the dashboard-builder skill. + + Diagnostic only, deliberately outside ``strict_pass``: a mis-route already fails on the + missing draft, and gating on this would turn any future route that skips ``set_skills`` + into a false failure. Reported so such a failure reads as a mis-route rather than as a + model that simply did not draft. + """ + for tc in tool_call_events: + if tc.function_name != _SET_SKILLS_TOOL or not tc.result: + continue + result_data = tc.parsed_result() + if not isinstance(result_data, dict): + continue + payload = result_data.get("data", result_data) + skills = payload.get("skills_to_activate") if isinstance(payload, dict) else None + if isinstance(skills, list) and _BUILDER_SKILL in skills: + return True + return False + + +def _expected_type(expected_output: dict) -> str: + """The response part the expectation names. One source, so message and lookup agree.""" + return str(expected_output.get("type") or "dashboard") + + +def _extract_dashboard_part(chat_result: ChatResult, part_type: str) -> dict | None: + """The response part of ``part_type``, e.g. ``dashboard``. + + That type is in the SSE client's known-part set but has no dedicated accumulator, so it + arrives in ``unhandled_parts`` verbatim and is read back by type. Taken from the end for + the same reason ``_extract_draft_result`` does: a turn that drafts and then refines must + be read as the state it left behind, not the one it passed through. + """ + for part in reversed(chat_result.unhandled_parts): + if isinstance(part, dict) and part.get("type") == part_type: + return part + return None + + +def _widgets_of(dashboard: dict) -> list[dict]: + """Every widget of the draft, flattened across tabs and sections. + + Charts are matched dashboard-wide on purpose: the expectation names charts, not a layout, + so which tab one lands on is the agent's call. The date filter is deliberately the + opposite -- ``_check_date_range`` requires *every* tab to match. That is defensive rather + than observed: ``draft_dashboard`` copies one filter onto every tab unconditionally, so a + draft cannot currently disagree with itself across tabs, and the per-tab check is there to + notice if that ever stops being true. + + Only ``tabs`` is read. That is sound for a draft, which ``draft_dashboard`` always builds + tabbed, but an AAC v2 document carries its layout in root ``sections`` instead, so an + editing dataset cannot reuse this as it stands. + """ + return [ + widget + for tab in dashboard.get("tabs") or [] + for section in tab.get("sections") or [] + for widget in section.get("widgets") or [] + ] + + +def _check_visualizations(widgets: list[dict], new_ids: set[str], expected: list[dict]) -> tuple[list[str], list[str]]: + """Match every expected chart against the draft; extra charts the agent added are allowed. + + Returns ``(failures, title_notes)``. + + An entry with an ``id`` is an existing chart and is matched on that id alone: the id is + the chart's identity, while the widget title is ``title_override or fallback_title`` and + the override is the model's own choice, so requiring it adds flake without signal. A + title that differs is still reported, as a note rather than a failure. + + ``id: null`` marks a chart the agent had to author, whose real id is generated at run + time. It has no identity to match on, so the title *is* the match — taken among the + authored charts only, without which an existing chart of the same title would satisfy it. + """ + failures: list[str] = [] + title_notes: list[str] = [] + for exp in expected: + exp_id = exp.get("id") + exp_title = str(exp.get("title") or "") + if exp_id is not None: + matches = [w for w in widgets if w.get("visualization") == exp_id] + if not matches: + failures.append(f"missing existing chart id={exp_id!r} title={exp_title!r}") + elif not any(_norm(str(w.get("title") or "")) == _norm(exp_title) for w in matches): + # Every title the id appears under: the same chart can be placed twice, and + # naming only the first would hide the rest. + titles = ", ".join(repr(w.get("title")) for w in matches) + title_notes.append(f"chart {exp_id!r} is titled {titles}, expected {exp_title!r}") + elif not any( + w.get("visualization") in new_ids and _norm(str(w.get("title") or "")) == _norm(exp_title) for w in widgets + ): + failures.append( + f"missing authored chart title={exp_title!r}" + + ("" if new_ids else " (the response listed no authored charts)") + ) + return failures, title_notes + + +def _check_references(widgets: list[dict], known_ids: set[str]) -> list[str]: + """Report widgets whose id the response's references never carried. + + Diagnostic, never a gate. gen-ai rejects a draft naming an unresolvable visualization + before it can succeed (``reject_unverified``), so this cannot catch an invented id. What + it does catch is reference building having degraded to an empty list on the way out -- + a platform fault, which must not be scored as the agent's. + """ + notes: list[str] = [] + for widget in widgets: + viz_id = widget.get("visualization") + if viz_id is not None and viz_id not in known_ids: + notes.append(f"the response's references do not carry {viz_id!r} (widget {widget.get('title')!r})") + return notes + + +def _check_date_range(dashboard: dict, expected: dict | None) -> list[str]: + """The date filter of every tab must match the expectation. + + A draft always carries a date filter, so ``expected is None`` (all time) means the tab + filter has neither bound — absent keys, not null ones. + """ + tabs = dashboard.get("tabs") or [] + if not tabs: + return ["draft has no tabs to read a date filter from"] + failures: list[str] = [] + for tab in tabs: + tab_id = tab.get("id") + date_filter = (tab.get("filters") or {}).get("date") or {} + if expected is None: + if "from" in date_filter or "to" in date_filter: + failures.append( + f"tab {tab_id!r}: expected all time, got from={date_filter.get('from')!r} " + f"to={date_filter.get('to')!r}" + ) + continue + failures.extend( + f"tab {tab_id!r}: date {key} expected {expected.get(key)!r}, got {date_filter.get(key)!r}" + for key in ("granularity", "from", "to") + if date_filter.get(key) != expected.get(key) + ) + return failures + + +def _date_range_text(date_range: dict | None) -> str: + """Render a date range the way a user would say it, for the simulated reply. + + Raises: + ValueError: the range is a shape no case has needed yet. Deliberately loud — a + silently mis-rendered range would be answered as a passing run on the wrong + dashboard. + """ + if date_range is None: + return "all time" + granularity = str(date_range.get("granularity", "")).lower() + start, end = date_range.get("from"), date_range.get("to") + if start == 0 and end == 0: + return f"this {granularity}" + if start == -1 and end == -1: + return f"last {granularity}" + raise ValueError(f"unsupported date_range for the simulated user: {date_range!r}") + + +def build_simulated_reply(expected_output: dict) -> str: + """Build the reply the simulated user sends when the agent asks back instead of drafting. + + Deterministic and LLM-free: the only things the agent still needs are the charts and the + date range, and both are already in the expectation. A fixed reply also keeps a failure + attributable to the agent rather than to a simulated user that phrased things differently + each run. + """ + visualizations = expected_output.get("visualizations") or [] + existing = [str(v.get("title", "")) for v in visualizations if v.get("id") is not None] + authored = [str(v.get("title", "")) for v in visualizations if v.get("id") is None] + + segments: list[str] = [] + if existing: + segments.append(", ".join(existing)) + segments.extend(f'a new chart titled "{title}"' for title in authored) + + return ( + f"Please use these charts: {', and '.join(segments)}. " + f"Date range: {_date_range_text(expected_output.get('date_range'))}. " + f"Anything else is up to you. Please create the dashboard now." + ) + + +def _min_new_visualizations(expected_output: dict) -> int: + """Lower bound on the number of charts the agent must author. + + Read here and nowhere else, so the value the fixture check accepts and the value the + score compares against cannot drift apart. + + Raises: + ValueError: the bound is not a number, or is negative — a negative bound would make + the comparison true for any count and switch the check off without saying so. + """ + raw = expected_output.get("min_new_visualizations") or 0 + try: + value = int(raw) + except (TypeError, ValueError) as exc: + raise ValueError(f"min_new_visualizations is not a number: {raw!r}") from exc + if value < 0: + raise ValueError(f"min_new_visualizations must not be negative, got {value}") + return value + + +def _validate_expectation(expected_output: dict) -> None: + """Reject a fixture the run could not score meaningfully, before the first API call. + + An empty ``visualizations`` would make every chart check pass vacuously, and a date range + the simulated user cannot phrase would otherwise surface only on the branch where the + agent asks back — passing or crashing depending on what the model chose to do that run. + + Raises: + ValueError: the expectation is unusable. + """ + if not expected_output.get("visualizations"): + raise ValueError("expected_output lists no visualizations; every chart check would pass vacuously") + _date_range_text(expected_output.get("date_range")) + _min_new_visualizations(expected_output) + + +@dataclass +class DashboardEvaluation: + """Per-run outcome of the dashboard-skill checks. + + ``strict_checks`` is what the run is scored on, and every entry in it is something the + agent decided. ``skill_activated`` and ``references_carried`` are reported beside it but + never gate: the first can be false on a route that pins skills instead of calling + ``set_skills``, and the second reflects gen-ai's reference building rather than the + agent's choices. ``notes`` carries the same kind of observation for titles. + """ + + drafted: bool + part_present: bool + charts_matched: bool + date_range_correct: bool + new_visualizations_met: bool + skill_activated: bool = False + references_carried: bool = True + titles_matched: bool = True + failures: list[str] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + + @property + def strict_pass(self) -> bool: + return all(self.strict_checks.values()) + + @property + def strict_checks(self) -> dict[str, bool]: + return { + "dashboard_drafted": self.drafted, + "dashboard_part_present": self.part_present, + "charts_matched": self.charts_matched, + "date_range_correct": self.date_range_correct, + "new_visualizations_met": self.new_visualizations_met, + } + + @property + def diagnostics(self) -> dict[str, bool]: + """Observed but not scored — see the class docstring. + + ``titles_matched`` is here rather than in the gate so the decision to stop failing on + a model-chosen title stays reviewable: without a score, nothing would record how often + it happens. The two chart-level observations are omitted when no dashboard part was + read, because reporting them as true would claim references and titles were fine on a + run that produced neither. + """ + observed = {"dashboard_skill_activated": self.skill_activated} + if self.part_present: + observed["references_carried"] = self.references_carried + observed["titles_matched"] = self.titles_matched + return observed + + +@dataclass +class DashboardRunResult: + """Outcome of one K-run conversation for dashboard creation.""" + + conversation_id: str + evaluation: DashboardEvaluation + draft_result: dict | None = None + dashboard_part: dict | None = None + total_turns: int = 0 + total_steps: int = 0 + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None + tool_call_events: list[ToolCallEvent] = field(default_factory=list) + reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) + timings: PhaseTimings = field(default_factory=PhaseTimings) + + +@dataclass +class AgenticDashboardSummary: + """Aggregated outcome of K runs for dashboard creation.""" + + run_results: list[DashboardRunResult] + pass_at_k: bool + pass_power_k: bool + best: DashboardRunResult + + +def evaluate_dashboard_draft( + draft_result: dict | None, + dashboard_part: dict | None, + expected_output: dict, + skill_activated: bool, +) -> DashboardEvaluation: + """Score one drafted dashboard against its expectation. + + Pure — no network, no conversation state — so the assertion logic is unit-testable + without an agent. + """ + if draft_result is None: + return DashboardEvaluation( + drafted=False, + part_present=False, + charts_matched=False, + date_range_correct=False, + new_visualizations_met=False, + skill_activated=skill_activated, + failures=["the agent never produced a successful draft_dashboard call"], + ) + + min_new = _min_new_visualizations(expected_output) + actual_new = draft_result.get("new_visualization_count") + if isinstance(actual_new, int): + new_met = actual_new >= min_new + # Naming the shortfall, never "expected at least 0" -- see the envelope branch below. + new_failures = [] if new_met else [f"expected at least {min_new} authored chart(s), tool reported {actual_new}"] + else: + # Not a shortfall: the tool result no longer carries the key. Said plainly, because + # "expected at least 0 authored chart(s), tool reported None" reads as a broken test + # and would send whoever is on nightly duty looking at the model instead. + new_met = False + new_failures = [f"the draft result carries no new_visualization_count (got {actual_new!r})"] + + if dashboard_part is None: + return DashboardEvaluation( + drafted=True, + part_present=False, + charts_matched=False, + date_range_correct=False, + new_visualizations_met=new_met, + skill_activated=skill_activated, + failures=[ + f"the response carries no {_expected_type(expected_output)!r} part", + *new_failures, + ], + ) + + dashboard = dashboard_part.get("dashboard") or {} + references = dashboard_part.get("references") or {} + new_ids = {v.get("id") for v in references.get("new_visualizations") or [] if v.get("id")} + known_ids = {v.get("id") for v in references.get("visualizations") or [] if v.get("id")} | new_ids + widgets = _widgets_of(dashboard) + + chart_failures, title_notes = _check_visualizations(widgets, new_ids, expected_output.get("visualizations") or []) + reference_notes = _check_references(widgets, known_ids) + date_failures = _check_date_range(dashboard, expected_output.get("date_range")) + + return DashboardEvaluation( + drafted=True, + part_present=True, + charts_matched=not chart_failures, + date_range_correct=not date_failures, + new_visualizations_met=new_met, + skill_activated=skill_activated, + references_carried=not reference_notes, + titles_matched=not title_notes, + failures=[*chart_failures, *date_failures, *new_failures], + notes=[*title_notes, *reference_notes], + ) + + +def _execute_single_dashboard_run( + client: ChatClient, + conversation_id: str, + question: str, + expected_output: dict, + max_iterations: int, +) -> DashboardRunResult: + """Drive one conversation until the agent drafts a dashboard, then evaluate it. + + Nothing is cleaned up on the way out by design: gen-ai keeps the draft and any authored + chart in conversation state and persists neither until a user saves from the UI. + """ + expected_type = str(expected_output.get("type") or "dashboard") + draft_result: dict | None = None + dashboard_part: dict | None = None + turns = 0 + steps = 0 + current_question = question + reasoning_steps: list[str] = [] + response_id: str | None = None + all_tool_call_events: list[ToolCallEvent] = [] + all_reasoning_step_events: list[ReasoningStepEvent] = [] + timings = PhaseTimings() + turn_offset = 0.0 + tool_index_offset = 0 + reasoning_index_offset = 0 + + for iteration in range(max_iterations): + turns += 1 + agent_started = time.monotonic() + chat_result = client.send_message(conversation_id, current_question) + agent_elapsed = time.monotonic() - agent_started + timings.agent_s += agent_elapsed + reasoning_steps.extend(chat_result.reasoning_steps or []) + response_id = chat_result.response_id or response_id + turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( + chat_result, + turn_offset=turn_offset, + tool_index_offset=tool_index_offset, + reasoning_index_offset=reasoning_index_offset, + ) + all_tool_call_events.extend(chat_result.tool_call_events or []) + all_reasoning_step_events.extend(chat_result.reasoning_step_events or []) + steps += chat_result.reasoning_step_count + + candidate = _extract_draft_result(chat_result.tool_call_events or []) + if candidate is not None: + log_timer( + f"[timer] dashboard_skill {conversation_id} GoodData turn {turns} complete after " + f"{agent_elapsed:.2f}s; draft received" + ) + draft_result = candidate + dashboard_part = _extract_dashboard_part(chat_result, expected_type) + break + + response_text = (chat_result.text_response or "").strip() or render_answer_text(chat_result) + if not response_text and not chat_result.tool_call_events: + break + if iteration >= max_iterations - 1: + break + log_timer( + f"[timer] dashboard_skill {conversation_id} GoodData turn {turns} complete after " + f"{agent_elapsed:.2f}s; answering with the expected charts and date range" + ) + current_question = build_simulated_reply(expected_output) + + return DashboardRunResult( + conversation_id=conversation_id, + evaluation=evaluate_dashboard_draft( + draft_result, + dashboard_part, + expected_output, + _builder_skill_activated(all_tool_call_events), + ), + draft_result=draft_result, + dashboard_part=dashboard_part, + total_turns=turns, + total_steps=steps, + reasoning_steps=reasoning_steps, + response_id=response_id, + tool_call_events=all_tool_call_events, + reasoning_step_events=all_reasoning_step_events, + timings=timings, + ) + + +def run_agentic_dashboard_skill( + host: str, + token: str, + workspace_id: str, + question: str, + expected_output: dict, + k: int = _DEFAULT_K, + max_iterations: int = _DEFAULT_MAX_ITERATIONS, + initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, +) -> AgenticDashboardSummary: + """Run the dashboard-skill agentic evaluation K times and return a summary. + + Raises: + ValueError: the fixture is unusable — see ``_validate_expectation``. + """ + _validate_expectation(expected_output) + run_results: list[DashboardRunResult] = [] + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) + + try: + conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() + try: + run_results.append( + _execute_single_dashboard_run(client, conv_id_0, question, expected_output, max_iterations) + ) + finally: + if initial_conversation_id is None: # only delete conversations we created + client.delete_conversation(conv_id_0) + + for _ in range(1, k): + conv_id = client.create_conversation() + try: + run_results.append( + _execute_single_dashboard_run(client, conv_id, question, expected_output, max_iterations) + ) + finally: + client.delete_conversation(conv_id) + finally: + client.close() + + pass_at_k = any(r.evaluation.strict_pass for r in run_results) + pass_power_k = all(r.evaluation.strict_pass for r in run_results) + best = max(run_results, key=lambda r: sum(r.evaluation.strict_checks.values())) + return AgenticDashboardSummary( + run_results=run_results, + pass_at_k=pass_at_k, + pass_power_k=pass_power_k, + best=best, + ) + + +class DashboardSkillAssertionError(AgenticAssertionError): + """Raised when a dashboard-skill evaluation fails.""" + + +def evaluate_agentic_dashboard_skill( + host: str, + token: str, + workspace_id: str, + question: str, + expected_output: dict, + k: int = _DEFAULT_K, + max_iterations: int = _DEFAULT_MAX_ITERATIONS, + initial_conversation_id: str | None = None, + agent_id: str | None = None, + langfuse: object | None = None, + dataset_item_id: str = "", + dataset_name: str = "dashboard_skill", + run_timestamp: str | None = None, + model_version_override: str | None = None, + run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, + gate: EvalGate = DEFAULT_GATE, +) -> AgenticEvalOutcome: + """Run dashboard-skill evaluation, log to Langfuse, and raise on failure. + + Returns the best run's outcome on success; on failure the same values are attached to + the raised ``DashboardSkillAssertionError`` so callers can retrieve them either way. + + Raises: + DashboardSkillAssertionError: the gate did not pass. + ValueError: the fixture is unusable — see ``_validate_expectation``. Raised before + any request, so it means a fixture to fix rather than a result to read. + """ + langfuse, window_start = open_trace_window(langfuse) + summary = run_agentic_dashboard_skill( + host=host, + token=token, + workspace_id=workspace_id, + question=question, + expected_output=expected_output, + k=k, + max_iterations=max_iterations, + initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, + agent_id=agent_id, + ) + + if langfuse is not None and dataset_item_id: + # Pinned on the calling thread: a deferred poll must not widen its query window. + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate) + + for run_idx, run in enumerate(summary.run_results): + pt = ctx.trace(run.conversation_id) + strict_checks = run.evaluation.strict_checks + with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + for score_name, value in strict_checks.items(): + ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") + for name, value in run.evaluation.diagnostics.items(): + ctx.score(tid, name=name, value=float(value), data_type="BOOLEAN") + ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") + ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC") + log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) + ctx.quality( + tid, + strict_checks=strict_checks, + latency_sec=pt.latency if pt else None, + cost_usd=pt.total_cost if pt else None, + ) + + # Before the pass@K raise: a failing item's scores are the ones worth having. + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + conversation_ids=[r.conversation_id for r in summary.run_results], + window_start=window_start, + window_end=window_end, + suffix_runs=len(summary.run_results) > 1, + write_scores=_write_scores, + item_input=question, + ) + + item_timings = sum_timings([r.timings for r in summary.run_results]) + runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) + runs_effective = len(summary.run_results) + + best = summary.best + detail: dict[str, Any] = { + **best.evaluation.strict_checks, + **best.evaluation.diagnostics, + "failures": best.evaluation.failures, + "notes": best.evaluation.notes, + "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), + } + + if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_failure_note(gate, runs_passed, runs_effective) + skill_note = ( + "" + if best.evaluation.skill_activated or best.evaluation.drafted + else " No set_skills call activated dashboard_builder, so check the feature flag before the model." + ) + notes = "; ".join(best.evaluation.notes) + exc = DashboardSkillAssertionError( + f"Dashboard skill assertion failed. {gate_note}{skill_note} " + f"Checks: {best.evaluation.strict_checks}. " + f"Failures: {'; '.join(best.evaluation.failures) or 'none reported'}." + + (f" Notes (not scored): {notes}." if notes else "") + ) + exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id + exc.timings = item_timings + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = runs_effective + raise exc + return AgenticEvalOutcome( + runs_passed=runs_passed, + runs_effective=runs_effective, + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + timings=item_timings, + ) diff --git a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py new file mode 100644 index 000000000..0957690bd --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py @@ -0,0 +1,679 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +import json +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic.dashboard_skill import ( + DashboardSkillAssertionError, + _builder_skill_activated, + _date_range_text, + _extract_draft_result, + build_simulated_reply, + evaluate_agentic_dashboard_skill, + evaluate_dashboard_draft, + run_agentic_dashboard_skill, +) +from gooddata_eval.core.models import ChatResult, ToolCallEvent + +# Ids and titles are the ones the eval layout seeds; shapes are trimmed from real runs of +# `agent_dashboard_skill` against ecommerce_demo. +_TOTAL_CUSTOMERS = "9f2a657a-22e1-4791-8216-7a354bb8de5d" +_ACTIVE_CUSTOMERS = "dc8575f5-27e5-44be-a2dc-23d54b7777e7" +_RETURN_CUSTOMERS = "4cea4177-37c5-4196-8aab-c6bf60dc1f22" +_ACTIVITY_BY_HOUR = "f1027458-1123-4f4d-af20-0e21b1d5e009" +_NET_SALES = "700d732d-67e1-4658-a6e0-c6b8edbb6d49" + +_ALL_TIME_FILTER = {"type": "date_filter", "granularity": "MONTH"} +_THIS_YEAR_FILTER = {"to": 0, "from": 0, "type": "date_filter", "granularity": "YEAR"} +_LAST_MONTH_FILTER = {"to": -1, "from": -1, "type": "date_filter", "granularity": "MONTH"} + + +def _widget(title: str, visualization: str) -> dict: + return {"rows": 22, "columns": 6, "title": title, "visualization": visualization} + + +def _dashboard_part( + widgets: list[dict], + *, + date_filter: dict, + visualizations: list[str] | None = None, + new_visualizations: list[str] | None = None, + part_type: str = "dashboard", + tabs: list[dict] | None = None, +) -> dict: + default_tabs = [ + { + "id": "overview", + "title": "Overview", + "filters": {"date": date_filter}, + "sections": [{"title": "Customer KPIs", "widgets": widgets}], + } + ] + referenced = visualizations if visualizations is not None else [w["visualization"] for w in widgets] + return { + "type": part_type, + "dashboard": { + "id": "customer_overview_fafd7270", + "type": "dashboard", + "version": "3", + "title": "Customer Overview", + "tabs": tabs if tabs is not None else default_tabs, + }, + "saved_dashboard_id": None, + "references": { + "datasets": [], + "datedatasets": [], + "visualizations": [{"id": vid, "type": "headline_chart", "title": vid} for vid in referenced], + "new_visualizations": [{"id": vid, "type": "bar_chart", "title": vid} for vid in new_visualizations or []], + }, + } + + +def _draft_result(new_visualization_count: int = 0, widget_count: int = 2) -> dict: + return { + "status": "success", + "ref": "dash_1", + "widget_count": widget_count, + "new_visualization_count": new_visualization_count, + } + + +def _tool_call(name: str, result: dict | None = None, arguments: dict | None = None) -> dict: + return { + "functionName": name, + "functionArguments": "{}" if arguments is None else json.dumps(arguments), + "result": None if result is None else json.dumps(result), + } + + +def _chat_result( + *, tool_calls: list[dict] | None = None, parts: list[dict] | None = None, text: str = "" +) -> ChatResult: + return ChatResult.model_validate( + { + "textResponse": text, + "toolCallEvents": tool_calls or [], + "unhandledParts": parts or [], + "streamEnded": True, + } + ) + + +_DC05_EXPECTED = { + "type": "dashboard", + "visualizations": [ + {"id": _TOTAL_CUSTOMERS, "title": "Total Customers"}, + {"id": _ACTIVE_CUSTOMERS, "title": "Active Customers"}, + ], + "date_range": {"granularity": "YEAR", "from": 0, "to": 0}, + "min_new_visualizations": 0, +} + +_DC03_EXPECTED = { + "type": "dashboard", + "visualizations": [ + {"id": _ACTIVITY_BY_HOUR, "title": "Activity by Hour"}, + {"id": None, "title": "Net Orders by Customer Age"}, + ], + "date_range": None, + "min_new_visualizations": 1, +} + + +class TestExtractDraftResult: + def test_returns_the_successful_payload(self): + events = _as_events([_tool_call("draft_dashboard", _draft_result())]) + assert _extract_draft_result(events) == _draft_result() + + def test_a_retry_is_not_shadowed_by_the_earlier_failure(self): + events = _as_events( + [ + _tool_call("draft_dashboard", {"status": "error", "message": "unknown visualization"}), + _tool_call("draft_dashboard", _draft_result(widget_count=9)), + ] + ) + assert _extract_draft_result(events) == _draft_result(widget_count=9) + + def test_a_failed_only_turn_yields_nothing(self): + events = _as_events([_tool_call("draft_dashboard", {"status": "error"})]) + assert _extract_draft_result(events) is None + + def test_a_call_without_a_result_yields_nothing(self): + assert _extract_draft_result(_as_events([_tool_call("draft_dashboard")])) is None + + +class TestBuilderSkillActivated: + def test_reads_the_set_skills_result(self): + events = _as_events([_tool_call("set_skills", {"skills_to_activate": ["dashboard_builder"]})]) + assert _builder_skill_activated(events) is True + + def test_activation_alongside_other_skills_counts(self): + events = _as_events([_tool_call("set_skills", {"skills_to_activate": ["dashboard_builder", "search"]})]) + assert _builder_skill_activated(events) is True + + def test_a_different_skill_does_not_count(self): + events = _as_events([_tool_call("set_skills", {"skills_to_activate": ["visualization"]})]) + assert _builder_skill_activated(events) is False + + +class TestEvaluateDraft: + def test_passes_when_the_draft_carries_the_expected_charts(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert ev.strict_pass + assert ev.failures == [] + + def test_charts_the_agent_added_on_its_own_do_not_fail_the_case(self): + part = _dashboard_part( + [ + _widget("Total Customers", _TOTAL_CUSTOMERS), + _widget("Active Customers", _ACTIVE_CUSTOMERS), + _widget("Return Customers", _RETURN_CUSTOMERS), + ], + date_filter=_THIS_YEAR_FILTER, + ) + assert evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True).strict_pass + + def test_a_missing_expected_chart_fails(self): + part = _dashboard_part([_widget("Total Customers", _TOTAL_CUSTOMERS)], date_filter=_THIS_YEAR_FILTER) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert not ev.charts_matched + assert any("Active Customers" in f for f in ev.failures) + + def test_a_model_chosen_title_is_noted_rather_than_failed(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Customers (active)", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert ev.charts_matched + assert ev.strict_pass + assert any("Customers (active)" in n for n in ev.notes) + + def test_title_comparison_ignores_case_and_whitespace(self): + part = _dashboard_part( + [_widget("total customers", _TOTAL_CUSTOMERS), _widget("ACTIVE CUSTOMERS", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + assert evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True).charts_matched + + def test_an_authored_chart_title_is_not_matched_by_prefix(self): + """The layout holds both `Net Sales` and `Net Sales vs Orders`, so the authored-chart + branch -- where the title is the only identity -- must not accept a longer title.""" + authored_id = "net_sales_vs_orders_11aa22bb" + expected = { + "type": "dashboard", + "visualizations": [{"id": None, "title": "Net Sales"}], + "date_range": None, + "min_new_visualizations": 1, + } + part = _dashboard_part( + [_widget("Net Sales vs Orders", authored_id)], + date_filter=_ALL_TIME_FILTER, + visualizations=[], + new_visualizations=[authored_id], + ) + ev = evaluate_dashboard_draft(_draft_result(new_visualization_count=1), part, expected, skill_activated=True) + assert not ev.charts_matched + + +class TestAdHocCharts: + def test_an_authored_chart_is_matched_by_title(self): + adhoc_id = "net_orders_by_customer_age_87e41365" + part = _dashboard_part( + [_widget("Activity by Hour", _ACTIVITY_BY_HOUR), _widget("Net Orders by Customer Age", adhoc_id)], + date_filter=_ALL_TIME_FILTER, + visualizations=[_ACTIVITY_BY_HOUR], + new_visualizations=[adhoc_id], + ) + ev = evaluate_dashboard_draft( + _draft_result(new_visualization_count=1), part, _DC03_EXPECTED, skill_activated=True + ) + assert ev.strict_pass + + def test_an_existing_chart_of_the_same_title_does_not_satisfy_an_authored_one(self): + existing_id = "b1d1f1a0-0000-4000-8000-000000000000" + part = _dashboard_part( + [_widget("Activity by Hour", _ACTIVITY_BY_HOUR), _widget("Net Orders by Customer Age", existing_id)], + date_filter=_ALL_TIME_FILTER, + visualizations=[_ACTIVITY_BY_HOUR, existing_id], + new_visualizations=[], + ) + ev = evaluate_dashboard_draft( + _draft_result(new_visualization_count=1), part, _DC03_EXPECTED, skill_activated=True + ) + assert not ev.charts_matched + + def test_a_missing_count_is_reported_as_an_envelope_change(self): + """Never "expected at least 0 authored chart(s), tool reported None" -- an assertion + demanding zero that still fails reads as a broken test, and it would fail every + fixture whose min_new_visualizations is 0 at once.""" + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_draft( + {"status": "success", "ref": "dash_1", "widget_count": 2}, part, _DC05_EXPECTED, skill_activated=True + ) + assert not ev.new_visualizations_met + assert ev.failures == ["the draft result carries no new_visualization_count (got None)"] + + def test_a_real_shortfall_names_the_numbers(self): + adhoc_id = "net_orders_by_customer_age_87e41365" + part = _dashboard_part( + [_widget("Activity by Hour", _ACTIVITY_BY_HOUR), _widget("Net Orders by Customer Age", adhoc_id)], + date_filter=_ALL_TIME_FILTER, + visualizations=[_ACTIVITY_BY_HOUR], + new_visualizations=[adhoc_id], + ) + ev = evaluate_dashboard_draft(_draft_result(new_visualization_count=0), part, _DC03_EXPECTED, True) + assert any("expected at least 1 authored chart(s), tool reported 0" in f for f in ev.failures) + + def test_too_few_authored_charts_fails(self): + adhoc_id = "net_orders_by_customer_age_87e41365" + part = _dashboard_part( + [_widget("Activity by Hour", _ACTIVITY_BY_HOUR), _widget("Net Orders by Customer Age", adhoc_id)], + date_filter=_ALL_TIME_FILTER, + visualizations=[_ACTIVITY_BY_HOUR], + new_visualizations=[adhoc_id], + ) + ev = evaluate_dashboard_draft( + _draft_result(new_visualization_count=0), part, _DC03_EXPECTED, skill_activated=True + ) + assert not ev.new_visualizations_met + + +class TestReferences: + """gen-ai rejects a draft naming an unresolvable chart before it can succeed, so a widget + missing from the references means its reference building degraded -- a platform fault that + must not be scored as the agent's.""" + + def test_a_reference_the_response_did_not_carry_is_a_note(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + visualizations=[_TOTAL_CUSTOMERS], + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert ev.strict_pass + assert not ev.references_carried + assert any(_ACTIVE_CUSTOMERS in n for n in ev.notes) + + def test_references_degrading_to_empty_does_not_fail_an_existing_chart_case(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + visualizations=[], + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert ev.strict_pass + assert not ev.references_carried + + def test_an_authored_chart_still_needs_the_references_and_says_so(self): + part = _dashboard_part( + [_widget("Activity by Hour", _ACTIVITY_BY_HOUR), _widget("Net Orders by Customer Age", "adhoc_1")], + date_filter=_ALL_TIME_FILTER, + visualizations=[], + new_visualizations=[], + ) + ev = evaluate_dashboard_draft(_draft_result(new_visualization_count=1), part, _DC03_EXPECTED, True) + assert not ev.charts_matched + assert any("listed no authored charts" in f for f in ev.failures) + + +class TestDateRange: + def test_all_time_accepts_a_filter_without_bounds(self): + expected = {**_DC03_EXPECTED, "visualizations": [], "min_new_visualizations": 0} + part = _dashboard_part([], date_filter=_ALL_TIME_FILTER, visualizations=[]) + assert evaluate_dashboard_draft(_draft_result(), part, expected, skill_activated=True).date_range_correct + + def test_all_time_rejects_a_bounded_filter(self): + expected = {**_DC03_EXPECTED, "visualizations": [], "min_new_visualizations": 0} + part = _dashboard_part([], date_filter=_THIS_YEAR_FILTER, visualizations=[]) + assert not evaluate_dashboard_draft(_draft_result(), part, expected, skill_activated=True).date_range_correct + + def test_a_bounded_range_must_match_every_field(self): + expected = { + **_DC05_EXPECTED, + "visualizations": [], + "date_range": {"granularity": "MONTH", "from": -1, "to": -1}, + } + part = _dashboard_part([], date_filter=_LAST_MONTH_FILTER, visualizations=[]) + assert evaluate_dashboard_draft(_draft_result(), part, expected, skill_activated=True).date_range_correct + + def test_a_wrong_granularity_fails(self): + expected = {**_DC05_EXPECTED, "visualizations": []} + part = _dashboard_part([], date_filter={"to": 0, "from": 0, "granularity": "MONTH"}, visualizations=[]) + assert not evaluate_dashboard_draft(_draft_result(), part, expected, skill_activated=True).date_range_correct + + def test_every_tab_is_checked(self): + expected = {**_DC05_EXPECTED, "visualizations": []} + tabs = [ + {"id": "a", "filters": {"date": _THIS_YEAR_FILTER}, "sections": []}, + {"id": "b", "filters": {"date": _ALL_TIME_FILTER}, "sections": []}, + ] + part = _dashboard_part([], date_filter=_THIS_YEAR_FILTER, visualizations=[], tabs=tabs) + ev = evaluate_dashboard_draft(_draft_result(), part, expected, skill_activated=True) + assert not ev.date_range_correct + assert any("'b'" in f for f in ev.failures) + + +class TestDiagnostics: + """Reported beside the gate so the decisions that moved them out of it stay reviewable.""" + + def test_a_model_chosen_title_is_scored_even_though_it_does_not_fail(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Customers (active)", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert ev.strict_pass + assert ev.diagnostics["titles_matched"] is False + + def test_a_clean_draft_reports_both_chart_level_observations(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert ev.diagnostics == { + "dashboard_skill_activated": True, + "references_carried": True, + "titles_matched": True, + } + + def test_a_run_without_a_part_claims_nothing_about_charts(self): + """Reporting these as true would say references and titles were fine on a run that + produced neither.""" + ev = evaluate_dashboard_draft(None, None, _DC05_EXPECTED, skill_activated=False) + assert ev.diagnostics == {"dashboard_skill_activated": False} + + def test_every_title_the_id_appears_under_is_named(self): + part = _dashboard_part( + [ + _widget("Customers (active)", _ACTIVE_CUSTOMERS), + _widget("Active users", _ACTIVE_CUSTOMERS), + _widget("Total Customers", _TOTAL_CUSTOMERS), + ], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + note = next(n for n in ev.notes if _ACTIVE_CUSTOMERS in n) + assert "Customers (active)" in note + assert "Active users" in note + + def test_a_title_placed_correctly_once_is_not_reported(self): + part = _dashboard_part( + [ + _widget("Active Customers", _ACTIVE_CUSTOMERS), + _widget("Active Customers (copy)", _ACTIVE_CUSTOMERS), + _widget("Total Customers", _TOTAL_CUSTOMERS), + ], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert ev.diagnostics["titles_matched"] is True + + +class TestMissingPieces: + def test_no_draft_reports_the_missing_draft(self): + ev = evaluate_dashboard_draft(None, None, _DC05_EXPECTED, skill_activated=False) + assert not ev.drafted + assert not ev.strict_pass + assert any("never produced a successful draft_dashboard" in f for f in ev.failures) + + def test_a_draft_without_its_part_reports_the_missing_part(self): + ev = evaluate_dashboard_draft(_draft_result(), None, _DC05_EXPECTED, skill_activated=True) + assert ev.drafted + assert not ev.part_present + assert any("'dashboard' part" in f for f in ev.failures) + + def test_a_patch_expectation_reads_the_patch_part(self): + expected = {**_DC05_EXPECTED, "type": "dashboardPatch", "visualizations": []} + part = _dashboard_part([], date_filter=_THIS_YEAR_FILTER, visualizations=[], part_type="dashboardPatch") + assert evaluate_dashboard_draft(_draft_result(), part, expected, skill_activated=True).part_present + + +class TestSimulatedReply: + def test_existing_charts_are_listed_verbatim(self): + expected = { + "visualizations": [ + {"id": _TOTAL_CUSTOMERS, "title": "Total Customers"}, + {"id": _ACTIVE_CUSTOMERS, "title": "Active Customers"}, + ], + "date_range": {"granularity": "YEAR", "from": 0, "to": 0}, + } + assert build_simulated_reply(expected) == ( + "Please use these charts: Total Customers, Active Customers. " + "Date range: this year. " + "Anything else is up to you. Please create the dashboard now." + ) + + def test_an_authored_chart_is_appended(self): + assert build_simulated_reply(_DC03_EXPECTED) == ( + 'Please use these charts: Activity by Hour, and a new chart titled "Net Orders by Customer Age". ' + "Date range: all time. " + "Anything else is up to you. Please create the dashboard now." + ) + + def test_an_authored_chart_alone_reads_naturally(self): + expected = {"visualizations": [{"id": None, "title": "Net Orders by Customer Age"}], "date_range": None} + assert 'Please use these charts: a new chart titled "Net Orders by Customer Age".' in build_simulated_reply( + expected + ) + + @pytest.mark.parametrize( + ("date_range", "text"), + [ + (None, "all time"), + ({"granularity": "YEAR", "from": 0, "to": 0}, "this year"), + ({"granularity": "MONTH", "from": -1, "to": -1}, "last month"), + ], + ) + def test_date_range_wording(self, date_range, text): + assert _date_range_text(date_range) == text + + def test_an_unsupported_range_fails_loudly(self): + with pytest.raises(ValueError, match="unsupported date_range"): + _date_range_text({"granularity": "MONTH", "from": -3, "to": -1}) + + +class TestFixtureValidation: + """A hand-written fixture has to fail before the run spends an API call.""" + + def test_an_expectation_without_charts_is_rejected(self): + client = MagicMock() + with pytest.raises(ValueError, match="no visualizations"): + _run_with(client, {"type": "dashboard", "visualizations": [], "date_range": None}) + client.send_message.assert_not_called() + + def test_an_unparseable_minimum_is_rejected_up_front(self): + client = MagicMock() + with pytest.raises(ValueError, match="min_new_visualizations is not a number"): + _run_with(client, {**_DC05_EXPECTED, "min_new_visualizations": "two"}) + client.send_message.assert_not_called() + + def test_a_negative_minimum_is_rejected_up_front(self): + """A negative bound makes the comparison true for any count, switching the check off + without saying so -- which is the drift this validation exists to prevent.""" + client = MagicMock() + with pytest.raises(ValueError, match="must not be negative"): + _run_with(client, {**_DC05_EXPECTED, "min_new_visualizations": -1}) + client.send_message.assert_not_called() + + def test_an_unphrasable_date_range_is_rejected_up_front(self): + """Not left to the clarify branch: the same fixture would otherwise pass when the model + drafts straight away and crash when it asks, depending on the run.""" + client = MagicMock() + expected = {**_DC05_EXPECTED, "date_range": {"granularity": "MONTH", "from": -11, "to": 0}} + with pytest.raises(ValueError, match="unsupported date_range"): + _run_with(client, expected) + client.send_message.assert_not_called() + + +class TestRunLoop: + def test_a_first_turn_draft_never_reaches_the_simulated_user(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + client = MagicMock() + client.send_message.return_value = _chat_result( + tool_calls=[ + _tool_call("set_skills", {"skills_to_activate": ["dashboard_builder"]}), + _tool_call("draft_dashboard", _draft_result()), + ], + parts=[part], + ) + summary = _run_with(client, _DC05_EXPECTED) + assert client.send_message.call_count == 1 + assert summary.pass_at_k + assert summary.best.evaluation.skill_activated + + def test_a_clarifying_turn_is_answered_from_the_expectation(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + client = MagicMock() + client.send_message.side_effect = [ + _chat_result( + tool_calls=[_tool_call("set_skills", {"skills_to_activate": ["dashboard_builder"]})], + text="What time range should the dashboard default to?", + ), + _chat_result(tool_calls=[_tool_call("draft_dashboard", _draft_result())], parts=[part]), + ] + summary = _run_with(client, _DC05_EXPECTED) + assert client.send_message.call_count == 2 + assert client.send_message.call_args_list[1].args[1] == build_simulated_reply(_DC05_EXPECTED) + assert summary.pass_at_k + assert summary.best.total_turns == 2 + + def test_the_question_goes_once_and_every_later_turn_repeats_the_same_reply(self): + """The reply is built from the expectation, so it cannot change between turns. The + rounds after the first are slack for a turn that produced nothing, not an attempt to + say something new -- which is why they are allowed to be identical.""" + client = MagicMock() + client.send_message.return_value = _chat_result(text="Which charts would you like?") + _run_with(client, _DC05_EXPECTED, max_iterations=None) + sent = [call.args[1] for call in client.send_message.call_args_list] + assert sent[0] == "Could you create a dashboard?" + assert set(sent[1:]) == {build_simulated_reply(_DC05_EXPECTED)} + + def test_a_wasted_first_turn_still_leaves_the_reply_a_chance(self): + """The case this slack exists for: the opening turn comes back without a draft, and the + cap must not have been spent by it.""" + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + client = MagicMock() + client.send_message.side_effect = [ + _chat_result(text="Sorry, something went wrong on our side."), + _chat_result(text="Which charts would you like?"), + _chat_result(tool_calls=[_tool_call("draft_dashboard", _draft_result())], parts=[part]), + ] + summary = _run_with(client, _DC05_EXPECTED, max_iterations=None) + assert summary.pass_at_k + assert summary.best.total_turns == 3 + + def test_the_last_dashboard_part_of_a_turn_wins(self): + """Matches _extract_draft_result taking the last successful call: a turn that drafts and + then refines must be read as the state it left behind.""" + superseded = _dashboard_part([_widget("Total Customers", _TOTAL_CUSTOMERS)], date_filter=_THIS_YEAR_FILTER) + final = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + client = MagicMock() + client.send_message.return_value = _chat_result( + tool_calls=[_tool_call("draft_dashboard", _draft_result())], parts=[superseded, final] + ) + assert _run_with(client, _DC05_EXPECTED).pass_at_k + + def test_the_loop_is_capped(self): + client = MagicMock() + client.send_message.return_value = _chat_result(text="Which charts would you like?") + summary = _run_with(client, _DC05_EXPECTED, max_iterations=2) + assert client.send_message.call_count == 2 + assert not summary.pass_at_k + assert not summary.best.evaluation.drafted + + def test_an_empty_turn_stops_the_loop(self): + client = MagicMock() + client.send_message.return_value = _chat_result() + summary = _run_with(client, _DC05_EXPECTED, max_iterations=5) + assert client.send_message.call_count == 1 + assert not summary.pass_at_k + + def test_a_caller_supplied_conversation_is_not_deleted(self): + client = MagicMock() + client.send_message.return_value = _chat_result() + _run_with(client, _DC05_EXPECTED, initial_conversation_id="conv-1") + client.create_conversation.assert_not_called() + client.delete_conversation.assert_not_called() + + +class TestEvaluateEntryPoint: + def test_a_failing_run_raises_with_the_failures_attached(self): + client = MagicMock() + client.send_message.return_value = _chat_result(text="Which charts would you like?") + with pytest.raises(DashboardSkillAssertionError) as excinfo: + _evaluate_with(client, _DC05_EXPECTED) + message = str(excinfo.value) + assert "No set_skills call activated dashboard_builder" in message + assert excinfo.value.detail["dashboard_drafted"] is False + assert excinfo.value.conversation_id == "conv-1" + + def test_a_passing_run_returns_the_outcome(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + client = MagicMock() + client.send_message.return_value = _chat_result( + tool_calls=[ + _tool_call("set_skills", {"skills_to_activate": ["dashboard_builder"]}), + _tool_call("draft_dashboard", _draft_result()), + ], + parts=[part], + ) + outcome = _evaluate_with(client, _DC05_EXPECTED) + assert outcome.runs_passed == 1 + assert outcome.runs_effective == 1 + assert outcome.detail["charts_matched"] is True + + +def _as_events(raw: list[dict]) -> list[ToolCallEvent]: + return [ToolCallEvent.model_validate(e) for e in raw] + + +def _run_with(client, expected_output, *, max_iterations=7, initial_conversation_id="conv-1"): + """``max_iterations=None`` exercises the module default rather than overriding it.""" + kwargs = {} if max_iterations is None else {"max_iterations": max_iterations} + with patch("gooddata_eval.core.agentic.dashboard_skill.ChatClient", return_value=client): + return run_agentic_dashboard_skill( + host="https://example.invalid", + token="token", + workspace_id="ws", + question="Could you create a dashboard?", + expected_output=expected_output, + initial_conversation_id=initial_conversation_id, + **kwargs, + ) + + +def _evaluate_with(client, expected_output): + with patch("gooddata_eval.core.agentic.dashboard_skill.ChatClient", return_value=client): + return evaluate_agentic_dashboard_skill( + host="https://example.invalid", + token="token", + workspace_id="ws", + question="Could you create a dashboard?", + expected_output=expected_output, + initial_conversation_id="conv-1", + ) diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 77086d101..e6efaf09f 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -80,6 +80,11 @@ def test_dispatch_agentic_omits_agent_id_by_default(): ("agentic_visualization", {"visualization": _MIN_VIZ}, "evaluate_agentic_visualization"), ("agentic_metric_skill", {"maql": "SELECT {metric/spend}"}, "evaluate_agentic_metric_skill"), ("agentic_alert_skill", {"Operator": "GREATER_THAN", "Threshold": 100}, "evaluate_agentic_alert_skill"), + ( + "agentic_dashboard_skill", + {"type": "dashboard", "visualizations": [], "date_range": None, "min_new_visualizations": 0}, + "evaluate_agentic_dashboard_skill", + ), ("agentic_search", {"tool_call": {"function_arguments": {}}}, "evaluate_agentic_search_tool"), ("agentic_general_question", "What is X?", "evaluate_agentic_general_question"), ("agentic_guardrail", "Ignore prior instructions", "evaluate_agentic_guardrail"), diff --git a/packages/gooddata-eval/tests/test_trace_linker.py b/packages/gooddata-eval/tests/test_trace_linker.py index f490effd6..25e8a5956 100644 --- a/packages/gooddata-eval/tests/test_trace_linker.py +++ b/packages/gooddata-eval/tests/test_trace_linker.py @@ -114,6 +114,7 @@ def test_run_trace_link_inline_runs_the_task_on_the_calling_thread(): ("guardrail", "evaluate_agentic_guardrail"), ("metric_skill", "evaluate_agentic_metric_skill"), ("alert_skill", "evaluate_agentic_alert_skill"), + ("dashboard_skill", "evaluate_agentic_dashboard_skill"), ("search_tool", "evaluate_agentic_search_tool"), ("visualization", "evaluate_agentic_visualization"), ("kda_skill", "evaluate_agentic_kda_skill"),