diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 129f4c5cf..3e0a2ef50 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -13,6 +13,7 @@ from gooddata_eval.core.agentic._trace_linker import BackgroundTraceLinker, SubmitTraceLink, run_trace_link_inline from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation +from gooddata_eval.core.agentic.dashboard_skill import evaluate_agentic_dashboard_skill from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill @@ -40,6 +41,7 @@ class _LfKw(TypedDict, total=False): "agentic_visualization", # experimental: expected_output.expected_outputs (multi-candidate) "agentic_metric_skill", "agentic_alert_skill", + "agentic_dashboard_skill", "agentic_search", "agentic_general_question", "agentic_guardrail", @@ -85,6 +87,10 @@ class _LfKw(TypedDict, total=False): # create_key_driver_analysis with no cleanup, and while the evaluator only ever reads that # call's ARGUMENTS -- never a created object id -- whether the platform persists anything is # unverified. Move it to the allowlist once someone confirms it does not. +# +# agentic_dashboard_skill is absent by default rather than by evidence: gen-ai holds the draft and +# any chart it authors in conversation state and writes neither until a user saves from the UI, so +# it is a candidate for the allowlist once the dataset has runs behind it. WORKSPACE_MUTATING_TEST_KINDS = frozenset(AGENTIC_TEST_KINDS) - PARALLEL_SAFE_TEST_KINDS @@ -183,6 +189,18 @@ def _dispatch_agentic( agent_id=agent_id, **lf_kw, ) + elif kind == "agentic_dashboard_skill": + return evaluate_agentic_dashboard_skill( + host=host, + token=token, + workspace_id=workspace_id, + question=item.question, + expected_output=eo if isinstance(eo, dict) else {}, + k=k, + gate=gate, + agent_id=agent_id, + **lf_kw, + ) elif kind == "agentic_alert_skill": return evaluate_agentic_alert_skill( host=host, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py index 89e93dde8..3d74a4b9f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py @@ -16,6 +16,14 @@ evaluate_agentic_conversation, run_agentic_conversation, ) +from gooddata_eval.core.agentic.dashboard_skill import ( + AgenticDashboardSummary, + DashboardEvaluation, + DashboardRunResult, + DashboardSkillAssertionError, + evaluate_agentic_dashboard_skill, + run_agentic_dashboard_skill, +) from gooddata_eval.core.agentic.general_question import ( AgenticGeneralQuestionSummary, GeneralQuestionAssertionError, @@ -62,6 +70,7 @@ __all__ = [ "AgenticAlertSummary", + "AgenticDashboardSummary", "AgenticGeneralQuestionSummary", "AgenticGuardrailSummary", "AgenticKdaSummary", @@ -74,6 +83,9 @@ "ConversationAssertionError", "ConversationFixture", "ConversationResult", + "DashboardEvaluation", + "DashboardRunResult", + "DashboardSkillAssertionError", "GeneralQuestionAssertionError", "GeneralQuestionResult", "GuardrailAssertionError", @@ -91,6 +103,7 @@ "VisualizationAssertionError", "evaluate_agentic_alert_skill", "evaluate_agentic_conversation", + "evaluate_agentic_dashboard_skill", "evaluate_agentic_general_question", "evaluate_agentic_guardrail", "evaluate_agentic_kda_skill", @@ -99,6 +112,7 @@ "evaluate_agentic_visualization", "run_agentic_alert_skill", "run_agentic_conversation", + "run_agentic_dashboard_skill", "run_agentic_general_question", "run_agentic_guardrail", "run_agentic_kda_skill", diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py new file mode 100644 index 000000000..81eeabe4f --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py @@ -0,0 +1,742 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Agentic dashboard-skill evaluation runner.""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any + +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) +from gooddata_eval.core.chat.render import render_answer_text +from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ChatResult, + ReasoningStepEvent, + ToolCallEvent, + build_latency_breakdown, + shift_and_index_events, +) +from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings + +_DEFAULT_K = 1 +# Matches kda_skill and visualization. The reply this loop sends is built from the expectation +# and is byte-identical every turn, so the extra rounds are not there to say anything new -- +# they are slack for a turn that produced nothing. A turn can come back empty or answer without +# drafting for reasons that are not the model's, and with a tight cap that one wasted turn +# spends the only reply the case had; the run then fails as "no draft" when nothing was wrong +# with the agent. Fewer than metric_skill's seven because those rounds do carry new content: +# its reply is generated per turn by an LLM. +# +# The cost of the slack is that four turns at ChatClient's 300s read timeout exceed the 720s +# per-test timeout gdc-nas derives for a k=1 dataset, so a run that stalls on every turn is cut +# short by pytest rather than by this loop. Measured turns are nowhere near that -- a two-turn +# case completes in about four minutes -- so the ceiling only binds when something is already +# badly wrong. +_DEFAULT_MAX_ITERATIONS = 4 + +_DRAFT_TOOL = "draft_dashboard" +_SET_SKILLS_TOOL = "set_skills" +_BUILDER_SKILL = "dashboard_builder" + + +def _norm(text: str) -> str: + """Case- and whitespace-insensitive form used for every title comparison.""" + return " ".join(text.split()).casefold() + + +def _extract_draft_result(tool_call_events: list[ToolCallEvent]) -> dict | None: + """Result payload of the ``draft_dashboard`` call that produced a draft. + + Takes the most recent *successful* call: when the agent retries after a rejected + draft, the earlier failed attempt must not shadow the one that worked. + """ + for tc in reversed(tool_call_events): + if tc.function_name != _DRAFT_TOOL or not tc.result: + continue + result_data = tc.parsed_result() + if not isinstance(result_data, dict): + continue + payload = result_data.get("data", result_data) + if not isinstance(payload, dict) or payload.get("status") != "success": + continue + return payload + return None + + +def _builder_skill_activated(tool_call_events: list[ToolCallEvent]) -> bool: + """Whether any ``set_skills`` result activated the dashboard-builder skill. + + Diagnostic only, deliberately outside ``strict_pass``: a mis-route already fails on the + missing draft, and gating on this would turn any future route that skips ``set_skills`` + into a false failure. Reported so such a failure reads as a mis-route rather than as a + model that simply did not draft. + """ + for tc in tool_call_events: + if tc.function_name != _SET_SKILLS_TOOL or not tc.result: + continue + result_data = tc.parsed_result() + if not isinstance(result_data, dict): + continue + payload = result_data.get("data", result_data) + skills = payload.get("skills_to_activate") if isinstance(payload, dict) else None + if isinstance(skills, list) and _BUILDER_SKILL in skills: + return True + return False + + +def _expected_type(expected_output: dict) -> str: + """The response part the expectation names. One source, so message and lookup agree.""" + return str(expected_output.get("type") or "dashboard") + + +def _extract_dashboard_part(chat_result: ChatResult, part_type: str) -> dict | None: + """The response part of ``part_type``, e.g. ``dashboard``. + + That type is in the SSE client's known-part set but has no dedicated accumulator, so it + arrives in ``unhandled_parts`` verbatim and is read back by type. Taken from the end for + the same reason ``_extract_draft_result`` does: a turn that drafts and then refines must + be read as the state it left behind, not the one it passed through. + """ + for part in reversed(chat_result.unhandled_parts): + if isinstance(part, dict) and part.get("type") == part_type: + return part + return None + + +def _widgets_of(dashboard: dict) -> list[dict]: + """Every widget of the draft, flattened across tabs and sections. + + Charts are matched dashboard-wide on purpose: the expectation names charts, not a layout, + so which tab one lands on is the agent's call. The date filter is deliberately the + opposite -- ``_check_date_range`` requires *every* tab to match. That is defensive rather + than observed: ``draft_dashboard`` copies one filter onto every tab unconditionally, so a + draft cannot currently disagree with itself across tabs, and the per-tab check is there to + notice if that ever stops being true. + + Only ``tabs`` is read. That is sound for a draft, which ``draft_dashboard`` always builds + tabbed, but an AAC v2 document carries its layout in root ``sections`` instead, so an + editing dataset cannot reuse this as it stands. + """ + return [ + widget + for tab in dashboard.get("tabs") or [] + for section in tab.get("sections") or [] + for widget in section.get("widgets") or [] + ] + + +def _check_visualizations(widgets: list[dict], new_ids: set[str], expected: list[dict]) -> tuple[list[str], list[str]]: + """Match every expected chart against the draft; extra charts the agent added are allowed. + + Returns ``(failures, title_notes)``. + + An entry with an ``id`` is an existing chart and is matched on that id alone: the id is + the chart's identity, while the widget title is ``title_override or fallback_title`` and + the override is the model's own choice, so requiring it adds flake without signal. A + title that differs is still reported, as a note rather than a failure. + + ``id: null`` marks a chart the agent had to author, whose real id is generated at run + time. It has no identity to match on, so the title *is* the match — taken among the + authored charts only, without which an existing chart of the same title would satisfy it. + """ + failures: list[str] = [] + title_notes: list[str] = [] + for exp in expected: + exp_id = exp.get("id") + exp_title = str(exp.get("title") or "") + if exp_id is not None: + matches = [w for w in widgets if w.get("visualization") == exp_id] + if not matches: + failures.append(f"missing existing chart id={exp_id!r} title={exp_title!r}") + elif not any(_norm(str(w.get("title") or "")) == _norm(exp_title) for w in matches): + # Every title the id appears under: the same chart can be placed twice, and + # naming only the first would hide the rest. + titles = ", ".join(repr(w.get("title")) for w in matches) + title_notes.append(f"chart {exp_id!r} is titled {titles}, expected {exp_title!r}") + elif not any( + w.get("visualization") in new_ids and _norm(str(w.get("title") or "")) == _norm(exp_title) for w in widgets + ): + failures.append( + f"missing authored chart title={exp_title!r}" + + ("" if new_ids else " (the response listed no authored charts)") + ) + return failures, title_notes + + +def _check_references(widgets: list[dict], known_ids: set[str]) -> list[str]: + """Report widgets whose id the response's references never carried. + + Diagnostic, never a gate. gen-ai rejects a draft naming an unresolvable visualization + before it can succeed (``reject_unverified``), so this cannot catch an invented id. What + it does catch is reference building having degraded to an empty list on the way out -- + a platform fault, which must not be scored as the agent's. + """ + notes: list[str] = [] + for widget in widgets: + viz_id = widget.get("visualization") + if viz_id is not None and viz_id not in known_ids: + notes.append(f"the response's references do not carry {viz_id!r} (widget {widget.get('title')!r})") + return notes + + +def _check_date_range(dashboard: dict, expected: dict | None) -> list[str]: + """The date filter of every tab must match the expectation. + + A draft always carries a date filter, so ``expected is None`` (all time) means the tab + filter has neither bound — absent keys, not null ones. + """ + tabs = dashboard.get("tabs") or [] + if not tabs: + return ["draft has no tabs to read a date filter from"] + failures: list[str] = [] + for tab in tabs: + tab_id = tab.get("id") + date_filter = (tab.get("filters") or {}).get("date") or {} + if expected is None: + if "from" in date_filter or "to" in date_filter: + failures.append( + f"tab {tab_id!r}: expected all time, got from={date_filter.get('from')!r} " + f"to={date_filter.get('to')!r}" + ) + continue + failures.extend( + f"tab {tab_id!r}: date {key} expected {expected.get(key)!r}, got {date_filter.get(key)!r}" + for key in ("granularity", "from", "to") + if date_filter.get(key) != expected.get(key) + ) + return failures + + +def _date_range_text(date_range: dict | None) -> str: + """Render a date range the way a user would say it, for the simulated reply. + + Raises: + ValueError: the range is a shape no case has needed yet. Deliberately loud — a + silently mis-rendered range would be answered as a passing run on the wrong + dashboard. + """ + if date_range is None: + return "all time" + granularity = str(date_range.get("granularity", "")).lower() + start, end = date_range.get("from"), date_range.get("to") + if start == 0 and end == 0: + return f"this {granularity}" + if start == -1 and end == -1: + return f"last {granularity}" + raise ValueError(f"unsupported date_range for the simulated user: {date_range!r}") + + +def build_simulated_reply(expected_output: dict) -> str: + """Build the reply the simulated user sends when the agent asks back instead of drafting. + + Deterministic and LLM-free: the only things the agent still needs are the charts and the + date range, and both are already in the expectation. A fixed reply also keeps a failure + attributable to the agent rather than to a simulated user that phrased things differently + each run. + """ + visualizations = expected_output.get("visualizations") or [] + existing = [str(v.get("title", "")) for v in visualizations if v.get("id") is not None] + authored = [str(v.get("title", "")) for v in visualizations if v.get("id") is None] + + segments: list[str] = [] + if existing: + segments.append(", ".join(existing)) + segments.extend(f'a new chart titled "{title}"' for title in authored) + + return ( + f"Please use these charts: {', and '.join(segments)}. " + f"Date range: {_date_range_text(expected_output.get('date_range'))}. " + f"Anything else is up to you. Please create the dashboard now." + ) + + +def _min_new_visualizations(expected_output: dict) -> int: + """Lower bound on the number of charts the agent must author. + + Read here and nowhere else, so the value the fixture check accepts and the value the + score compares against cannot drift apart. + + Raises: + ValueError: the bound is not a number, or is negative — a negative bound would make + the comparison true for any count and switch the check off without saying so. + """ + raw = expected_output.get("min_new_visualizations") or 0 + try: + value = int(raw) + except (TypeError, ValueError) as exc: + raise ValueError(f"min_new_visualizations is not a number: {raw!r}") from exc + if value < 0: + raise ValueError(f"min_new_visualizations must not be negative, got {value}") + return value + + +def _validate_expectation(expected_output: dict) -> None: + """Reject a fixture the run could not score meaningfully, before the first API call. + + An empty ``visualizations`` would make every chart check pass vacuously, and a date range + the simulated user cannot phrase would otherwise surface only on the branch where the + agent asks back — passing or crashing depending on what the model chose to do that run. + + Raises: + ValueError: the expectation is unusable. + """ + if not expected_output.get("visualizations"): + raise ValueError("expected_output lists no visualizations; every chart check would pass vacuously") + _date_range_text(expected_output.get("date_range")) + _min_new_visualizations(expected_output) + + +@dataclass +class DashboardEvaluation: + """Per-run outcome of the dashboard-skill checks. + + ``strict_checks`` is what the run is scored on, and every entry in it is something the + agent decided. ``skill_activated`` and ``references_carried`` are reported beside it but + never gate: the first can be false on a route that pins skills instead of calling + ``set_skills``, and the second reflects gen-ai's reference building rather than the + agent's choices. ``notes`` carries the same kind of observation for titles. + """ + + drafted: bool + part_present: bool + charts_matched: bool + date_range_correct: bool + new_visualizations_met: bool + skill_activated: bool = False + references_carried: bool = True + titles_matched: bool = True + failures: list[str] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + + @property + def strict_pass(self) -> bool: + return all(self.strict_checks.values()) + + @property + def strict_checks(self) -> dict[str, bool]: + return { + "dashboard_drafted": self.drafted, + "dashboard_part_present": self.part_present, + "charts_matched": self.charts_matched, + "date_range_correct": self.date_range_correct, + "new_visualizations_met": self.new_visualizations_met, + } + + @property + def diagnostics(self) -> dict[str, bool]: + """Observed but not scored — see the class docstring. + + ``titles_matched`` is here rather than in the gate so the decision to stop failing on + a model-chosen title stays reviewable: without a score, nothing would record how often + it happens. The two chart-level observations are omitted when no dashboard part was + read, because reporting them as true would claim references and titles were fine on a + run that produced neither. + """ + observed = {"dashboard_skill_activated": self.skill_activated} + if self.part_present: + observed["references_carried"] = self.references_carried + observed["titles_matched"] = self.titles_matched + return observed + + +@dataclass +class DashboardRunResult: + """Outcome of one K-run conversation for dashboard creation.""" + + conversation_id: str + evaluation: DashboardEvaluation + draft_result: dict | None = None + dashboard_part: dict | None = None + total_turns: int = 0 + total_steps: int = 0 + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None + tool_call_events: list[ToolCallEvent] = field(default_factory=list) + reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) + timings: PhaseTimings = field(default_factory=PhaseTimings) + + +@dataclass +class AgenticDashboardSummary: + """Aggregated outcome of K runs for dashboard creation.""" + + run_results: list[DashboardRunResult] + pass_at_k: bool + pass_power_k: bool + best: DashboardRunResult + + +def evaluate_dashboard_draft( + draft_result: dict | None, + dashboard_part: dict | None, + expected_output: dict, + skill_activated: bool, +) -> DashboardEvaluation: + """Score one drafted dashboard against its expectation. + + Pure — no network, no conversation state — so the assertion logic is unit-testable + without an agent. + """ + if draft_result is None: + return DashboardEvaluation( + drafted=False, + part_present=False, + charts_matched=False, + date_range_correct=False, + new_visualizations_met=False, + skill_activated=skill_activated, + failures=["the agent never produced a successful draft_dashboard call"], + ) + + min_new = _min_new_visualizations(expected_output) + actual_new = draft_result.get("new_visualization_count") + if isinstance(actual_new, int): + new_met = actual_new >= min_new + # Naming the shortfall, never "expected at least 0" -- see the envelope branch below. + new_failures = [] if new_met else [f"expected at least {min_new} authored chart(s), tool reported {actual_new}"] + else: + # Not a shortfall: the tool result no longer carries the key. Said plainly, because + # "expected at least 0 authored chart(s), tool reported None" reads as a broken test + # and would send whoever is on nightly duty looking at the model instead. + new_met = False + new_failures = [f"the draft result carries no new_visualization_count (got {actual_new!r})"] + + if dashboard_part is None: + return DashboardEvaluation( + drafted=True, + part_present=False, + charts_matched=False, + date_range_correct=False, + new_visualizations_met=new_met, + skill_activated=skill_activated, + failures=[ + f"the response carries no {_expected_type(expected_output)!r} part", + *new_failures, + ], + ) + + dashboard = dashboard_part.get("dashboard") or {} + references = dashboard_part.get("references") or {} + new_ids = {v.get("id") for v in references.get("new_visualizations") or [] if v.get("id")} + known_ids = {v.get("id") for v in references.get("visualizations") or [] if v.get("id")} | new_ids + widgets = _widgets_of(dashboard) + + chart_failures, title_notes = _check_visualizations(widgets, new_ids, expected_output.get("visualizations") or []) + reference_notes = _check_references(widgets, known_ids) + date_failures = _check_date_range(dashboard, expected_output.get("date_range")) + + return DashboardEvaluation( + drafted=True, + part_present=True, + charts_matched=not chart_failures, + date_range_correct=not date_failures, + new_visualizations_met=new_met, + skill_activated=skill_activated, + references_carried=not reference_notes, + titles_matched=not title_notes, + failures=[*chart_failures, *date_failures, *new_failures], + notes=[*title_notes, *reference_notes], + ) + + +def _execute_single_dashboard_run( + client: ChatClient, + conversation_id: str, + question: str, + expected_output: dict, + max_iterations: int, +) -> DashboardRunResult: + """Drive one conversation until the agent drafts a dashboard, then evaluate it. + + Nothing is cleaned up on the way out by design: gen-ai keeps the draft and any authored + chart in conversation state and persists neither until a user saves from the UI. + """ + expected_type = str(expected_output.get("type") or "dashboard") + draft_result: dict | None = None + dashboard_part: dict | None = None + turns = 0 + steps = 0 + current_question = question + reasoning_steps: list[str] = [] + response_id: str | None = None + all_tool_call_events: list[ToolCallEvent] = [] + all_reasoning_step_events: list[ReasoningStepEvent] = [] + timings = PhaseTimings() + turn_offset = 0.0 + tool_index_offset = 0 + reasoning_index_offset = 0 + + for iteration in range(max_iterations): + turns += 1 + agent_started = time.monotonic() + chat_result = client.send_message(conversation_id, current_question) + agent_elapsed = time.monotonic() - agent_started + timings.agent_s += agent_elapsed + reasoning_steps.extend(chat_result.reasoning_steps or []) + response_id = chat_result.response_id or response_id + turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( + chat_result, + turn_offset=turn_offset, + tool_index_offset=tool_index_offset, + reasoning_index_offset=reasoning_index_offset, + ) + all_tool_call_events.extend(chat_result.tool_call_events or []) + all_reasoning_step_events.extend(chat_result.reasoning_step_events or []) + steps += chat_result.reasoning_step_count + + candidate = _extract_draft_result(chat_result.tool_call_events or []) + if candidate is not None: + log_timer( + f"[timer] dashboard_skill {conversation_id} GoodData turn {turns} complete after " + f"{agent_elapsed:.2f}s; draft received" + ) + draft_result = candidate + dashboard_part = _extract_dashboard_part(chat_result, expected_type) + break + + response_text = (chat_result.text_response or "").strip() or render_answer_text(chat_result) + if not response_text and not chat_result.tool_call_events: + break + if iteration >= max_iterations - 1: + break + log_timer( + f"[timer] dashboard_skill {conversation_id} GoodData turn {turns} complete after " + f"{agent_elapsed:.2f}s; answering with the expected charts and date range" + ) + current_question = build_simulated_reply(expected_output) + + return DashboardRunResult( + conversation_id=conversation_id, + evaluation=evaluate_dashboard_draft( + draft_result, + dashboard_part, + expected_output, + _builder_skill_activated(all_tool_call_events), + ), + draft_result=draft_result, + dashboard_part=dashboard_part, + total_turns=turns, + total_steps=steps, + reasoning_steps=reasoning_steps, + response_id=response_id, + tool_call_events=all_tool_call_events, + reasoning_step_events=all_reasoning_step_events, + timings=timings, + ) + + +def run_agentic_dashboard_skill( + host: str, + token: str, + workspace_id: str, + question: str, + expected_output: dict, + k: int = _DEFAULT_K, + max_iterations: int = _DEFAULT_MAX_ITERATIONS, + initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, +) -> AgenticDashboardSummary: + """Run the dashboard-skill agentic evaluation K times and return a summary. + + Raises: + ValueError: the fixture is unusable — see ``_validate_expectation``. + """ + _validate_expectation(expected_output) + run_results: list[DashboardRunResult] = [] + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) + + try: + conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() + try: + run_results.append( + _execute_single_dashboard_run(client, conv_id_0, question, expected_output, max_iterations) + ) + finally: + if initial_conversation_id is None: # only delete conversations we created + client.delete_conversation(conv_id_0) + + for _ in range(1, k): + conv_id = client.create_conversation() + try: + run_results.append( + _execute_single_dashboard_run(client, conv_id, question, expected_output, max_iterations) + ) + finally: + client.delete_conversation(conv_id) + finally: + client.close() + + pass_at_k = any(r.evaluation.strict_pass for r in run_results) + pass_power_k = all(r.evaluation.strict_pass for r in run_results) + best = max(run_results, key=lambda r: sum(r.evaluation.strict_checks.values())) + return AgenticDashboardSummary( + run_results=run_results, + pass_at_k=pass_at_k, + pass_power_k=pass_power_k, + best=best, + ) + + +class DashboardSkillAssertionError(AgenticAssertionError): + """Raised when a dashboard-skill evaluation fails.""" + + +def evaluate_agentic_dashboard_skill( + host: str, + token: str, + workspace_id: str, + question: str, + expected_output: dict, + k: int = _DEFAULT_K, + max_iterations: int = _DEFAULT_MAX_ITERATIONS, + initial_conversation_id: str | None = None, + agent_id: str | None = None, + langfuse: object | None = None, + dataset_item_id: str = "", + dataset_name: str = "dashboard_skill", + run_timestamp: str | None = None, + model_version_override: str | None = None, + run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, + gate: EvalGate = DEFAULT_GATE, +) -> AgenticEvalOutcome: + """Run dashboard-skill evaluation, log to Langfuse, and raise on failure. + + Returns the best run's outcome on success; on failure the same values are attached to + the raised ``DashboardSkillAssertionError`` so callers can retrieve them either way. + + Raises: + DashboardSkillAssertionError: the gate did not pass. + ValueError: the fixture is unusable — see ``_validate_expectation``. Raised before + any request, so it means a fixture to fix rather than a result to read. + """ + langfuse, window_start = open_trace_window(langfuse) + summary = run_agentic_dashboard_skill( + host=host, + token=token, + workspace_id=workspace_id, + question=question, + expected_output=expected_output, + k=k, + max_iterations=max_iterations, + initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, + agent_id=agent_id, + ) + + if langfuse is not None and dataset_item_id: + # Pinned on the calling thread: a deferred poll must not widen its query window. + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate) + + for run_idx, run in enumerate(summary.run_results): + pt = ctx.trace(run.conversation_id) + strict_checks = run.evaluation.strict_checks + with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + for score_name, value in strict_checks.items(): + ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") + for name, value in run.evaluation.diagnostics.items(): + ctx.score(tid, name=name, value=float(value), data_type="BOOLEAN") + ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") + ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC") + log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) + ctx.quality( + tid, + strict_checks=strict_checks, + latency_sec=pt.latency if pt else None, + cost_usd=pt.total_cost if pt else None, + ) + + # Before the pass@K raise: a failing item's scores are the ones worth having. + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + conversation_ids=[r.conversation_id for r in summary.run_results], + window_start=window_start, + window_end=window_end, + suffix_runs=len(summary.run_results) > 1, + write_scores=_write_scores, + item_input=question, + ) + + item_timings = sum_timings([r.timings for r in summary.run_results]) + runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) + runs_effective = len(summary.run_results) + + best = summary.best + detail: dict[str, Any] = { + **best.evaluation.strict_checks, + **best.evaluation.diagnostics, + "failures": best.evaluation.failures, + "notes": best.evaluation.notes, + "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), + } + + if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_failure_note(gate, runs_passed, runs_effective) + skill_note = ( + "" + if best.evaluation.skill_activated or best.evaluation.drafted + else " No set_skills call activated dashboard_builder, so check the feature flag before the model." + ) + notes = "; ".join(best.evaluation.notes) + exc = DashboardSkillAssertionError( + f"Dashboard skill assertion failed. {gate_note}{skill_note} " + f"Checks: {best.evaluation.strict_checks}. " + f"Failures: {'; '.join(best.evaluation.failures) or 'none reported'}." + + (f" Notes (not scored): {notes}." if notes else "") + ) + exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id + exc.timings = item_timings + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = runs_effective + raise exc + return AgenticEvalOutcome( + runs_passed=runs_passed, + runs_effective=runs_effective, + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + timings=item_timings, + ) diff --git a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py new file mode 100644 index 000000000..0957690bd --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py @@ -0,0 +1,679 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +import json +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic.dashboard_skill import ( + DashboardSkillAssertionError, + _builder_skill_activated, + _date_range_text, + _extract_draft_result, + build_simulated_reply, + evaluate_agentic_dashboard_skill, + evaluate_dashboard_draft, + run_agentic_dashboard_skill, +) +from gooddata_eval.core.models import ChatResult, ToolCallEvent + +# Ids and titles are the ones the eval layout seeds; shapes are trimmed from real runs of +# `agent_dashboard_skill` against ecommerce_demo. +_TOTAL_CUSTOMERS = "9f2a657a-22e1-4791-8216-7a354bb8de5d" +_ACTIVE_CUSTOMERS = "dc8575f5-27e5-44be-a2dc-23d54b7777e7" +_RETURN_CUSTOMERS = "4cea4177-37c5-4196-8aab-c6bf60dc1f22" +_ACTIVITY_BY_HOUR = "f1027458-1123-4f4d-af20-0e21b1d5e009" +_NET_SALES = "700d732d-67e1-4658-a6e0-c6b8edbb6d49" + +_ALL_TIME_FILTER = {"type": "date_filter", "granularity": "MONTH"} +_THIS_YEAR_FILTER = {"to": 0, "from": 0, "type": "date_filter", "granularity": "YEAR"} +_LAST_MONTH_FILTER = {"to": -1, "from": -1, "type": "date_filter", "granularity": "MONTH"} + + +def _widget(title: str, visualization: str) -> dict: + return {"rows": 22, "columns": 6, "title": title, "visualization": visualization} + + +def _dashboard_part( + widgets: list[dict], + *, + date_filter: dict, + visualizations: list[str] | None = None, + new_visualizations: list[str] | None = None, + part_type: str = "dashboard", + tabs: list[dict] | None = None, +) -> dict: + default_tabs = [ + { + "id": "overview", + "title": "Overview", + "filters": {"date": date_filter}, + "sections": [{"title": "Customer KPIs", "widgets": widgets}], + } + ] + referenced = visualizations if visualizations is not None else [w["visualization"] for w in widgets] + return { + "type": part_type, + "dashboard": { + "id": "customer_overview_fafd7270", + "type": "dashboard", + "version": "3", + "title": "Customer Overview", + "tabs": tabs if tabs is not None else default_tabs, + }, + "saved_dashboard_id": None, + "references": { + "datasets": [], + "datedatasets": [], + "visualizations": [{"id": vid, "type": "headline_chart", "title": vid} for vid in referenced], + "new_visualizations": [{"id": vid, "type": "bar_chart", "title": vid} for vid in new_visualizations or []], + }, + } + + +def _draft_result(new_visualization_count: int = 0, widget_count: int = 2) -> dict: + return { + "status": "success", + "ref": "dash_1", + "widget_count": widget_count, + "new_visualization_count": new_visualization_count, + } + + +def _tool_call(name: str, result: dict | None = None, arguments: dict | None = None) -> dict: + return { + "functionName": name, + "functionArguments": "{}" if arguments is None else json.dumps(arguments), + "result": None if result is None else json.dumps(result), + } + + +def _chat_result( + *, tool_calls: list[dict] | None = None, parts: list[dict] | None = None, text: str = "" +) -> ChatResult: + return ChatResult.model_validate( + { + "textResponse": text, + "toolCallEvents": tool_calls or [], + "unhandledParts": parts or [], + "streamEnded": True, + } + ) + + +_DC05_EXPECTED = { + "type": "dashboard", + "visualizations": [ + {"id": _TOTAL_CUSTOMERS, "title": "Total Customers"}, + {"id": _ACTIVE_CUSTOMERS, "title": "Active Customers"}, + ], + "date_range": {"granularity": "YEAR", "from": 0, "to": 0}, + "min_new_visualizations": 0, +} + +_DC03_EXPECTED = { + "type": "dashboard", + "visualizations": [ + {"id": _ACTIVITY_BY_HOUR, "title": "Activity by Hour"}, + {"id": None, "title": "Net Orders by Customer Age"}, + ], + "date_range": None, + "min_new_visualizations": 1, +} + + +class TestExtractDraftResult: + def test_returns_the_successful_payload(self): + events = _as_events([_tool_call("draft_dashboard", _draft_result())]) + assert _extract_draft_result(events) == _draft_result() + + def test_a_retry_is_not_shadowed_by_the_earlier_failure(self): + events = _as_events( + [ + _tool_call("draft_dashboard", {"status": "error", "message": "unknown visualization"}), + _tool_call("draft_dashboard", _draft_result(widget_count=9)), + ] + ) + assert _extract_draft_result(events) == _draft_result(widget_count=9) + + def test_a_failed_only_turn_yields_nothing(self): + events = _as_events([_tool_call("draft_dashboard", {"status": "error"})]) + assert _extract_draft_result(events) is None + + def test_a_call_without_a_result_yields_nothing(self): + assert _extract_draft_result(_as_events([_tool_call("draft_dashboard")])) is None + + +class TestBuilderSkillActivated: + def test_reads_the_set_skills_result(self): + events = _as_events([_tool_call("set_skills", {"skills_to_activate": ["dashboard_builder"]})]) + assert _builder_skill_activated(events) is True + + def test_activation_alongside_other_skills_counts(self): + events = _as_events([_tool_call("set_skills", {"skills_to_activate": ["dashboard_builder", "search"]})]) + assert _builder_skill_activated(events) is True + + def test_a_different_skill_does_not_count(self): + events = _as_events([_tool_call("set_skills", {"skills_to_activate": ["visualization"]})]) + assert _builder_skill_activated(events) is False + + +class TestEvaluateDraft: + def test_passes_when_the_draft_carries_the_expected_charts(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert ev.strict_pass + assert ev.failures == [] + + def test_charts_the_agent_added_on_its_own_do_not_fail_the_case(self): + part = _dashboard_part( + [ + _widget("Total Customers", _TOTAL_CUSTOMERS), + _widget("Active Customers", _ACTIVE_CUSTOMERS), + _widget("Return Customers", _RETURN_CUSTOMERS), + ], + date_filter=_THIS_YEAR_FILTER, + ) + assert evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True).strict_pass + + def test_a_missing_expected_chart_fails(self): + part = _dashboard_part([_widget("Total Customers", _TOTAL_CUSTOMERS)], date_filter=_THIS_YEAR_FILTER) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert not ev.charts_matched + assert any("Active Customers" in f for f in ev.failures) + + def test_a_model_chosen_title_is_noted_rather_than_failed(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Customers (active)", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert ev.charts_matched + assert ev.strict_pass + assert any("Customers (active)" in n for n in ev.notes) + + def test_title_comparison_ignores_case_and_whitespace(self): + part = _dashboard_part( + [_widget("total customers", _TOTAL_CUSTOMERS), _widget("ACTIVE CUSTOMERS", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + assert evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True).charts_matched + + def test_an_authored_chart_title_is_not_matched_by_prefix(self): + """The layout holds both `Net Sales` and `Net Sales vs Orders`, so the authored-chart + branch -- where the title is the only identity -- must not accept a longer title.""" + authored_id = "net_sales_vs_orders_11aa22bb" + expected = { + "type": "dashboard", + "visualizations": [{"id": None, "title": "Net Sales"}], + "date_range": None, + "min_new_visualizations": 1, + } + part = _dashboard_part( + [_widget("Net Sales vs Orders", authored_id)], + date_filter=_ALL_TIME_FILTER, + visualizations=[], + new_visualizations=[authored_id], + ) + ev = evaluate_dashboard_draft(_draft_result(new_visualization_count=1), part, expected, skill_activated=True) + assert not ev.charts_matched + + +class TestAdHocCharts: + def test_an_authored_chart_is_matched_by_title(self): + adhoc_id = "net_orders_by_customer_age_87e41365" + part = _dashboard_part( + [_widget("Activity by Hour", _ACTIVITY_BY_HOUR), _widget("Net Orders by Customer Age", adhoc_id)], + date_filter=_ALL_TIME_FILTER, + visualizations=[_ACTIVITY_BY_HOUR], + new_visualizations=[adhoc_id], + ) + ev = evaluate_dashboard_draft( + _draft_result(new_visualization_count=1), part, _DC03_EXPECTED, skill_activated=True + ) + assert ev.strict_pass + + def test_an_existing_chart_of_the_same_title_does_not_satisfy_an_authored_one(self): + existing_id = "b1d1f1a0-0000-4000-8000-000000000000" + part = _dashboard_part( + [_widget("Activity by Hour", _ACTIVITY_BY_HOUR), _widget("Net Orders by Customer Age", existing_id)], + date_filter=_ALL_TIME_FILTER, + visualizations=[_ACTIVITY_BY_HOUR, existing_id], + new_visualizations=[], + ) + ev = evaluate_dashboard_draft( + _draft_result(new_visualization_count=1), part, _DC03_EXPECTED, skill_activated=True + ) + assert not ev.charts_matched + + def test_a_missing_count_is_reported_as_an_envelope_change(self): + """Never "expected at least 0 authored chart(s), tool reported None" -- an assertion + demanding zero that still fails reads as a broken test, and it would fail every + fixture whose min_new_visualizations is 0 at once.""" + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_draft( + {"status": "success", "ref": "dash_1", "widget_count": 2}, part, _DC05_EXPECTED, skill_activated=True + ) + assert not ev.new_visualizations_met + assert ev.failures == ["the draft result carries no new_visualization_count (got None)"] + + def test_a_real_shortfall_names_the_numbers(self): + adhoc_id = "net_orders_by_customer_age_87e41365" + part = _dashboard_part( + [_widget("Activity by Hour", _ACTIVITY_BY_HOUR), _widget("Net Orders by Customer Age", adhoc_id)], + date_filter=_ALL_TIME_FILTER, + visualizations=[_ACTIVITY_BY_HOUR], + new_visualizations=[adhoc_id], + ) + ev = evaluate_dashboard_draft(_draft_result(new_visualization_count=0), part, _DC03_EXPECTED, True) + assert any("expected at least 1 authored chart(s), tool reported 0" in f for f in ev.failures) + + def test_too_few_authored_charts_fails(self): + adhoc_id = "net_orders_by_customer_age_87e41365" + part = _dashboard_part( + [_widget("Activity by Hour", _ACTIVITY_BY_HOUR), _widget("Net Orders by Customer Age", adhoc_id)], + date_filter=_ALL_TIME_FILTER, + visualizations=[_ACTIVITY_BY_HOUR], + new_visualizations=[adhoc_id], + ) + ev = evaluate_dashboard_draft( + _draft_result(new_visualization_count=0), part, _DC03_EXPECTED, skill_activated=True + ) + assert not ev.new_visualizations_met + + +class TestReferences: + """gen-ai rejects a draft naming an unresolvable chart before it can succeed, so a widget + missing from the references means its reference building degraded -- a platform fault that + must not be scored as the agent's.""" + + def test_a_reference_the_response_did_not_carry_is_a_note(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + visualizations=[_TOTAL_CUSTOMERS], + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert ev.strict_pass + assert not ev.references_carried + assert any(_ACTIVE_CUSTOMERS in n for n in ev.notes) + + def test_references_degrading_to_empty_does_not_fail_an_existing_chart_case(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + visualizations=[], + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert ev.strict_pass + assert not ev.references_carried + + def test_an_authored_chart_still_needs_the_references_and_says_so(self): + part = _dashboard_part( + [_widget("Activity by Hour", _ACTIVITY_BY_HOUR), _widget("Net Orders by Customer Age", "adhoc_1")], + date_filter=_ALL_TIME_FILTER, + visualizations=[], + new_visualizations=[], + ) + ev = evaluate_dashboard_draft(_draft_result(new_visualization_count=1), part, _DC03_EXPECTED, True) + assert not ev.charts_matched + assert any("listed no authored charts" in f for f in ev.failures) + + +class TestDateRange: + def test_all_time_accepts_a_filter_without_bounds(self): + expected = {**_DC03_EXPECTED, "visualizations": [], "min_new_visualizations": 0} + part = _dashboard_part([], date_filter=_ALL_TIME_FILTER, visualizations=[]) + assert evaluate_dashboard_draft(_draft_result(), part, expected, skill_activated=True).date_range_correct + + def test_all_time_rejects_a_bounded_filter(self): + expected = {**_DC03_EXPECTED, "visualizations": [], "min_new_visualizations": 0} + part = _dashboard_part([], date_filter=_THIS_YEAR_FILTER, visualizations=[]) + assert not evaluate_dashboard_draft(_draft_result(), part, expected, skill_activated=True).date_range_correct + + def test_a_bounded_range_must_match_every_field(self): + expected = { + **_DC05_EXPECTED, + "visualizations": [], + "date_range": {"granularity": "MONTH", "from": -1, "to": -1}, + } + part = _dashboard_part([], date_filter=_LAST_MONTH_FILTER, visualizations=[]) + assert evaluate_dashboard_draft(_draft_result(), part, expected, skill_activated=True).date_range_correct + + def test_a_wrong_granularity_fails(self): + expected = {**_DC05_EXPECTED, "visualizations": []} + part = _dashboard_part([], date_filter={"to": 0, "from": 0, "granularity": "MONTH"}, visualizations=[]) + assert not evaluate_dashboard_draft(_draft_result(), part, expected, skill_activated=True).date_range_correct + + def test_every_tab_is_checked(self): + expected = {**_DC05_EXPECTED, "visualizations": []} + tabs = [ + {"id": "a", "filters": {"date": _THIS_YEAR_FILTER}, "sections": []}, + {"id": "b", "filters": {"date": _ALL_TIME_FILTER}, "sections": []}, + ] + part = _dashboard_part([], date_filter=_THIS_YEAR_FILTER, visualizations=[], tabs=tabs) + ev = evaluate_dashboard_draft(_draft_result(), part, expected, skill_activated=True) + assert not ev.date_range_correct + assert any("'b'" in f for f in ev.failures) + + +class TestDiagnostics: + """Reported beside the gate so the decisions that moved them out of it stay reviewable.""" + + def test_a_model_chosen_title_is_scored_even_though_it_does_not_fail(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Customers (active)", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert ev.strict_pass + assert ev.diagnostics["titles_matched"] is False + + def test_a_clean_draft_reports_both_chart_level_observations(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert ev.diagnostics == { + "dashboard_skill_activated": True, + "references_carried": True, + "titles_matched": True, + } + + def test_a_run_without_a_part_claims_nothing_about_charts(self): + """Reporting these as true would say references and titles were fine on a run that + produced neither.""" + ev = evaluate_dashboard_draft(None, None, _DC05_EXPECTED, skill_activated=False) + assert ev.diagnostics == {"dashboard_skill_activated": False} + + def test_every_title_the_id_appears_under_is_named(self): + part = _dashboard_part( + [ + _widget("Customers (active)", _ACTIVE_CUSTOMERS), + _widget("Active users", _ACTIVE_CUSTOMERS), + _widget("Total Customers", _TOTAL_CUSTOMERS), + ], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + note = next(n for n in ev.notes if _ACTIVE_CUSTOMERS in n) + assert "Customers (active)" in note + assert "Active users" in note + + def test_a_title_placed_correctly_once_is_not_reported(self): + part = _dashboard_part( + [ + _widget("Active Customers", _ACTIVE_CUSTOMERS), + _widget("Active Customers (copy)", _ACTIVE_CUSTOMERS), + _widget("Total Customers", _TOTAL_CUSTOMERS), + ], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_draft(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert ev.diagnostics["titles_matched"] is True + + +class TestMissingPieces: + def test_no_draft_reports_the_missing_draft(self): + ev = evaluate_dashboard_draft(None, None, _DC05_EXPECTED, skill_activated=False) + assert not ev.drafted + assert not ev.strict_pass + assert any("never produced a successful draft_dashboard" in f for f in ev.failures) + + def test_a_draft_without_its_part_reports_the_missing_part(self): + ev = evaluate_dashboard_draft(_draft_result(), None, _DC05_EXPECTED, skill_activated=True) + assert ev.drafted + assert not ev.part_present + assert any("'dashboard' part" in f for f in ev.failures) + + def test_a_patch_expectation_reads_the_patch_part(self): + expected = {**_DC05_EXPECTED, "type": "dashboardPatch", "visualizations": []} + part = _dashboard_part([], date_filter=_THIS_YEAR_FILTER, visualizations=[], part_type="dashboardPatch") + assert evaluate_dashboard_draft(_draft_result(), part, expected, skill_activated=True).part_present + + +class TestSimulatedReply: + def test_existing_charts_are_listed_verbatim(self): + expected = { + "visualizations": [ + {"id": _TOTAL_CUSTOMERS, "title": "Total Customers"}, + {"id": _ACTIVE_CUSTOMERS, "title": "Active Customers"}, + ], + "date_range": {"granularity": "YEAR", "from": 0, "to": 0}, + } + assert build_simulated_reply(expected) == ( + "Please use these charts: Total Customers, Active Customers. " + "Date range: this year. " + "Anything else is up to you. Please create the dashboard now." + ) + + def test_an_authored_chart_is_appended(self): + assert build_simulated_reply(_DC03_EXPECTED) == ( + 'Please use these charts: Activity by Hour, and a new chart titled "Net Orders by Customer Age". ' + "Date range: all time. " + "Anything else is up to you. Please create the dashboard now." + ) + + def test_an_authored_chart_alone_reads_naturally(self): + expected = {"visualizations": [{"id": None, "title": "Net Orders by Customer Age"}], "date_range": None} + assert 'Please use these charts: a new chart titled "Net Orders by Customer Age".' in build_simulated_reply( + expected + ) + + @pytest.mark.parametrize( + ("date_range", "text"), + [ + (None, "all time"), + ({"granularity": "YEAR", "from": 0, "to": 0}, "this year"), + ({"granularity": "MONTH", "from": -1, "to": -1}, "last month"), + ], + ) + def test_date_range_wording(self, date_range, text): + assert _date_range_text(date_range) == text + + def test_an_unsupported_range_fails_loudly(self): + with pytest.raises(ValueError, match="unsupported date_range"): + _date_range_text({"granularity": "MONTH", "from": -3, "to": -1}) + + +class TestFixtureValidation: + """A hand-written fixture has to fail before the run spends an API call.""" + + def test_an_expectation_without_charts_is_rejected(self): + client = MagicMock() + with pytest.raises(ValueError, match="no visualizations"): + _run_with(client, {"type": "dashboard", "visualizations": [], "date_range": None}) + client.send_message.assert_not_called() + + def test_an_unparseable_minimum_is_rejected_up_front(self): + client = MagicMock() + with pytest.raises(ValueError, match="min_new_visualizations is not a number"): + _run_with(client, {**_DC05_EXPECTED, "min_new_visualizations": "two"}) + client.send_message.assert_not_called() + + def test_a_negative_minimum_is_rejected_up_front(self): + """A negative bound makes the comparison true for any count, switching the check off + without saying so -- which is the drift this validation exists to prevent.""" + client = MagicMock() + with pytest.raises(ValueError, match="must not be negative"): + _run_with(client, {**_DC05_EXPECTED, "min_new_visualizations": -1}) + client.send_message.assert_not_called() + + def test_an_unphrasable_date_range_is_rejected_up_front(self): + """Not left to the clarify branch: the same fixture would otherwise pass when the model + drafts straight away and crash when it asks, depending on the run.""" + client = MagicMock() + expected = {**_DC05_EXPECTED, "date_range": {"granularity": "MONTH", "from": -11, "to": 0}} + with pytest.raises(ValueError, match="unsupported date_range"): + _run_with(client, expected) + client.send_message.assert_not_called() + + +class TestRunLoop: + def test_a_first_turn_draft_never_reaches_the_simulated_user(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + client = MagicMock() + client.send_message.return_value = _chat_result( + tool_calls=[ + _tool_call("set_skills", {"skills_to_activate": ["dashboard_builder"]}), + _tool_call("draft_dashboard", _draft_result()), + ], + parts=[part], + ) + summary = _run_with(client, _DC05_EXPECTED) + assert client.send_message.call_count == 1 + assert summary.pass_at_k + assert summary.best.evaluation.skill_activated + + def test_a_clarifying_turn_is_answered_from_the_expectation(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + client = MagicMock() + client.send_message.side_effect = [ + _chat_result( + tool_calls=[_tool_call("set_skills", {"skills_to_activate": ["dashboard_builder"]})], + text="What time range should the dashboard default to?", + ), + _chat_result(tool_calls=[_tool_call("draft_dashboard", _draft_result())], parts=[part]), + ] + summary = _run_with(client, _DC05_EXPECTED) + assert client.send_message.call_count == 2 + assert client.send_message.call_args_list[1].args[1] == build_simulated_reply(_DC05_EXPECTED) + assert summary.pass_at_k + assert summary.best.total_turns == 2 + + def test_the_question_goes_once_and_every_later_turn_repeats_the_same_reply(self): + """The reply is built from the expectation, so it cannot change between turns. The + rounds after the first are slack for a turn that produced nothing, not an attempt to + say something new -- which is why they are allowed to be identical.""" + client = MagicMock() + client.send_message.return_value = _chat_result(text="Which charts would you like?") + _run_with(client, _DC05_EXPECTED, max_iterations=None) + sent = [call.args[1] for call in client.send_message.call_args_list] + assert sent[0] == "Could you create a dashboard?" + assert set(sent[1:]) == {build_simulated_reply(_DC05_EXPECTED)} + + def test_a_wasted_first_turn_still_leaves_the_reply_a_chance(self): + """The case this slack exists for: the opening turn comes back without a draft, and the + cap must not have been spent by it.""" + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + client = MagicMock() + client.send_message.side_effect = [ + _chat_result(text="Sorry, something went wrong on our side."), + _chat_result(text="Which charts would you like?"), + _chat_result(tool_calls=[_tool_call("draft_dashboard", _draft_result())], parts=[part]), + ] + summary = _run_with(client, _DC05_EXPECTED, max_iterations=None) + assert summary.pass_at_k + assert summary.best.total_turns == 3 + + def test_the_last_dashboard_part_of_a_turn_wins(self): + """Matches _extract_draft_result taking the last successful call: a turn that drafts and + then refines must be read as the state it left behind.""" + superseded = _dashboard_part([_widget("Total Customers", _TOTAL_CUSTOMERS)], date_filter=_THIS_YEAR_FILTER) + final = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + client = MagicMock() + client.send_message.return_value = _chat_result( + tool_calls=[_tool_call("draft_dashboard", _draft_result())], parts=[superseded, final] + ) + assert _run_with(client, _DC05_EXPECTED).pass_at_k + + def test_the_loop_is_capped(self): + client = MagicMock() + client.send_message.return_value = _chat_result(text="Which charts would you like?") + summary = _run_with(client, _DC05_EXPECTED, max_iterations=2) + assert client.send_message.call_count == 2 + assert not summary.pass_at_k + assert not summary.best.evaluation.drafted + + def test_an_empty_turn_stops_the_loop(self): + client = MagicMock() + client.send_message.return_value = _chat_result() + summary = _run_with(client, _DC05_EXPECTED, max_iterations=5) + assert client.send_message.call_count == 1 + assert not summary.pass_at_k + + def test_a_caller_supplied_conversation_is_not_deleted(self): + client = MagicMock() + client.send_message.return_value = _chat_result() + _run_with(client, _DC05_EXPECTED, initial_conversation_id="conv-1") + client.create_conversation.assert_not_called() + client.delete_conversation.assert_not_called() + + +class TestEvaluateEntryPoint: + def test_a_failing_run_raises_with_the_failures_attached(self): + client = MagicMock() + client.send_message.return_value = _chat_result(text="Which charts would you like?") + with pytest.raises(DashboardSkillAssertionError) as excinfo: + _evaluate_with(client, _DC05_EXPECTED) + message = str(excinfo.value) + assert "No set_skills call activated dashboard_builder" in message + assert excinfo.value.detail["dashboard_drafted"] is False + assert excinfo.value.conversation_id == "conv-1" + + def test_a_passing_run_returns_the_outcome(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + client = MagicMock() + client.send_message.return_value = _chat_result( + tool_calls=[ + _tool_call("set_skills", {"skills_to_activate": ["dashboard_builder"]}), + _tool_call("draft_dashboard", _draft_result()), + ], + parts=[part], + ) + outcome = _evaluate_with(client, _DC05_EXPECTED) + assert outcome.runs_passed == 1 + assert outcome.runs_effective == 1 + assert outcome.detail["charts_matched"] is True + + +def _as_events(raw: list[dict]) -> list[ToolCallEvent]: + return [ToolCallEvent.model_validate(e) for e in raw] + + +def _run_with(client, expected_output, *, max_iterations=7, initial_conversation_id="conv-1"): + """``max_iterations=None`` exercises the module default rather than overriding it.""" + kwargs = {} if max_iterations is None else {"max_iterations": max_iterations} + with patch("gooddata_eval.core.agentic.dashboard_skill.ChatClient", return_value=client): + return run_agentic_dashboard_skill( + host="https://example.invalid", + token="token", + workspace_id="ws", + question="Could you create a dashboard?", + expected_output=expected_output, + initial_conversation_id=initial_conversation_id, + **kwargs, + ) + + +def _evaluate_with(client, expected_output): + with patch("gooddata_eval.core.agentic.dashboard_skill.ChatClient", return_value=client): + return evaluate_agentic_dashboard_skill( + host="https://example.invalid", + token="token", + workspace_id="ws", + question="Could you create a dashboard?", + expected_output=expected_output, + initial_conversation_id="conv-1", + ) diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 77086d101..e6efaf09f 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -80,6 +80,11 @@ def test_dispatch_agentic_omits_agent_id_by_default(): ("agentic_visualization", {"visualization": _MIN_VIZ}, "evaluate_agentic_visualization"), ("agentic_metric_skill", {"maql": "SELECT {metric/spend}"}, "evaluate_agentic_metric_skill"), ("agentic_alert_skill", {"Operator": "GREATER_THAN", "Threshold": 100}, "evaluate_agentic_alert_skill"), + ( + "agentic_dashboard_skill", + {"type": "dashboard", "visualizations": [], "date_range": None, "min_new_visualizations": 0}, + "evaluate_agentic_dashboard_skill", + ), ("agentic_search", {"tool_call": {"function_arguments": {}}}, "evaluate_agentic_search_tool"), ("agentic_general_question", "What is X?", "evaluate_agentic_general_question"), ("agentic_guardrail", "Ignore prior instructions", "evaluate_agentic_guardrail"), diff --git a/packages/gooddata-eval/tests/test_trace_linker.py b/packages/gooddata-eval/tests/test_trace_linker.py index f490effd6..25e8a5956 100644 --- a/packages/gooddata-eval/tests/test_trace_linker.py +++ b/packages/gooddata-eval/tests/test_trace_linker.py @@ -114,6 +114,7 @@ def test_run_trace_link_inline_runs_the_task_on_the_calling_thread(): ("guardrail", "evaluate_agentic_guardrail"), ("metric_skill", "evaluate_agentic_metric_skill"), ("alert_skill", "evaluate_agentic_alert_skill"), + ("dashboard_skill", "evaluate_agentic_dashboard_skill"), ("search_tool", "evaluate_agentic_search_tool"), ("visualization", "evaluate_agentic_visualization"), ("kda_skill", "evaluate_agentic_kda_skill"),