diff --git a/packages/gooddata-eval/AGENTS.md b/packages/gooddata-eval/AGENTS.md index 3769cfafb..b615bc592 100644 --- a/packages/gooddata-eval/AGENTS.md +++ b/packages/gooddata-eval/AGENTS.md @@ -27,6 +27,7 @@ experiment. The newest and most actively developed package in the repo. | `core/agentic/` | multi-turn agentic evaluation per capability, **plus** all Langfuse trace polling and linking (`_langfuse.py`, `_trace_linker.py`) | | `core/chat/` | SSE client for the agent's streaming chat endpoint | | `core/summary/` | HTTP client for the dedicated dashboard-summary endpoint — a single-shot chat backend, not reporting | +| `core/takeaway/` | HTTP client for the AI Publisher takeaway endpoint — a single-shot backend like `core/summary/`, keeping the whole response on `ChatResult.raw_response` — plus `evaluate_document_takeaway`, the K-run, gate and Langfuse entry point the Tavern shim calls | | `core/dataset/` | dataset format, loading, and `from_insights.py` — dataset generation from a workspace's real insights | | `core/evaluators/` | single-shot evaluators and their registry | | `core/langfuse/` | the whole Langfuse v4 client: `_env` (base URL + credentials), `otlp` (OTLP/JSON encoding), `experiment` (root-span construction, score targets), `item_scope` (an item's experiment scope sent to gen-ai as baggage, and which conversations joined), `observations` (trace reads), `client` (httpx calls), `sink` (single-shot results as experiments) | @@ -59,7 +60,8 @@ implementation, sometimes under different `test_kind` strings (`search_tool` vs `expected_output: Any`. `expected_output` is deliberately untyped — each evaluator parses its own shape. `test_kind` on the item is what labels the result, not the evaluator class, which is why `knowledge_question` can reuse `GeneralQuestionEvaluator` verbatim. -`dashboard_summary` items additionally need `summary_input`. +`dashboard_summary` items additionally need `summary_input`, `document_takeaway` items +`takeaway_input`. ## Running the pipeline @@ -155,8 +157,8 @@ Run this before guessing a `--model` string. | `GD_EVAL_JUDGE_MODEL` | judge model, same as `--judge-model` | | `GD_EVAL_AGENT_ID` | which agent to drive, same as `--agent-id` | | `LANGFUSE_HOST`, `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` | `--langfuse`, `--langfuse-dataset` | -| `GOODDATA_EVAL_CHAT_*` | SSE retry, backoff and timeout knobs | -| `GOODDATA_EVAL_TRACE_LABELS` | `key=value,...` labels stamped on every observation of a chat conversation via W3C baggage; `model_version` also sets the observation `version`, so set it only for a one-model run: every `--model` in one process gets the same labels. Keys must match `[A-Za-z0-9_-]+`; others are skipped with a warning. Dashboard-summary runs are not labelled | +| `GOODDATA_EVAL_CHAT_*` | retry and backoff knobs shared by the chat and takeaway clients (`core/_retry.py`), and the SSE turn and item timeouts | +| `GOODDATA_EVAL_TRACE_LABELS` | `key=value,...` labels stamped on every observation of a chat conversation via W3C baggage; `model_version` also sets the observation `version`, so set it only for a one-model run: every `--model` in one process gets the same labels. Keys must match `[A-Za-z0-9_-]+`; others are skipped with a warning. Dashboard-summary and document-takeaway runs are not labelled | | `GOODDATA_EVAL_JOIN_GENAI_TRACE` | on unless set to `0`, `false`, `no`, `off` or empty. An agentic item's chat turns ask gen-ai to open them under gd-eval's experiment root, so each item is one Langfuse trace with its scores written once. A gen-ai that drops caller baggage keeps today's separate traces; see the gotcha below | | `GD_EVAL_TIMERS` | same as `--timers` | @@ -181,12 +183,15 @@ That default is correct: agent tool calls create real server-side objects (metri Adding a kind to `PARALLEL_SAFE_TEST_KINDS` is a deliberate assertion that it is read-only, which nothing in the package can prove for you. -**The SSE client's retry predicate is load-bearing.** `core/chat/sse_client.py` retries -429/502/503/504 and `httpx.RemoteProtocolError` (a mid-stream disconnect) with exponential -backoff, and treats a `METADATA_SYNC_IN_PROGRESS` payload as transient. The -`RemoteProtocolError` case was added after it was confirmed live to contaminate a small -percentage of visualization runs with a hard fail and no retry. Narrowing that predicate -reintroduces the problem. +**The SSE client's retry predicate is load-bearing.** The backoff loop and the retryable +status set (429/500/502/503/504) live in `core/_retry.py`, shared by the chat and takeaway +clients and tuned by the same `GOODDATA_EVAL_CHAT_*` knobs. `core/chat/sse_client.py` +retries that status set and `httpx.RemoteProtocolError` (a mid-stream disconnect), and +treats a `METADATA_SYNC_IN_PROGRESS` payload as transient. The `RemoteProtocolError` case +was added after it was confirmed live to contaminate a small percentage of visualization +runs with a hard fail and no retry. Narrowing that predicate reintroduces the problem. The +takeaway client retries the same status set plus connection failures, but not a read +timeout: a slow page would spend the whole timeout again on every attempt. **Langfuse trace linking is deliberately off the item critical path.** Polling for trace ingestion has no pass/fail signal and inflates or misattributes per-item latency, so diff --git a/packages/gooddata-eval/Makefile b/packages/gooddata-eval/Makefile index d1c4d1b3f..59c2f679e 100644 --- a/packages/gooddata-eval/Makefile +++ b/packages/gooddata-eval/Makefile @@ -1,2 +1,6 @@ # (C) 2026 GoodData Corporation include ../../project_common.mk + +.PHONY: calibrate-takeaway +calibrate-takeaway: + uv run --package gooddata-eval --extra llm-judge python scripts/calibrate_takeaway_judge.py diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index 702d60cc3..c6df83470 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -132,8 +132,8 @@ gd-eval run \ | `--runs K` | `2` | Independent runs per item. | | `--gate` | `any` | Which verdict decides an item: `any` = pass@K (a run passing is enough), `power` = pass^K (every run must pass, so the verdict measures stability). Identical at `--runs 1`. Kinds that repeat K runs only — `power` is refused when the dataset also has non-agentic items or `agentic_conversation`, which are always decided on pass@K. | | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests — see *Concurrency and workspace safety* below. | -| `--judge-model MODEL` | `gpt-4o` | Model used for LLM-as-judge scoring — `agentic_general_question`, `agentic_guardrail`, `general_question`, `guardrail` and `dashboard_summary`. Also settable via `GD_EVAL_JUDGE_MODEL`. Two things to weigh before changing it: the gpt-5 family rejects `temperature=0`, so verdicts stop being reproducible (the run warns when this happens); and choosing the same model the agent runs means the judge grades its own family's output. | -| `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` items go through the summary endpoint, which has no such option. | +| `--judge-model MODEL` | `gpt-4o` | Model used for LLM-as-judge scoring — `agentic_general_question`, `agentic_guardrail`, `general_question`, `guardrail`, `dashboard_summary` and `document_takeaway`. Also settable via `GD_EVAL_JUDGE_MODEL`. Two things to weigh before changing it: the gpt-5 family rejects `temperature=0`, so verdicts stop being reproducible (the run warns when this happens); and choosing the same model the agent runs means the judge grades its own family's output. | +| `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` and `document_takeaway` items go through their own endpoints, which have no such option. | **Concurrency and workspace safety.** Agentic kinds that create workspace objects (`agentic_metric_skill`, `agentic_alert_skill`, `agentic_conversation`, `agentic_kda_skill`) always run one at a @@ -707,7 +707,7 @@ A dataset is a folder of `.json` files, one per question: ``` Supported `test_kind` values: `visualization`, `metric_skill`, `alert_skill`, -`search_tool`, `general_question`, `guardrail`, `dashboard_summary`. +`search_tool`, `general_question`, `guardrail`, `dashboard_summary`, `document_takeaway`. ### `dashboard_summary` items @@ -746,6 +746,64 @@ The `expected_output` rubric: Each criterion is scored independently by the LLM judge, so `quality_score` is the fraction of satisfied criteria. +### `document_takeaway` items + +Takeaway items call the AI Publisher takeaway endpoint +(`POST /api/v1/ai/workspaces/{ws}/takeaway`) with a `takeaway_input` block, authored in +snake_case and sent as the endpoint's camelCase body. In a Langfuse dataset the block goes +in the item's `metadata` (or its `input` object), and `input` can stay a plain case label; +an item carrying `takeaway_input` is a `document_takeaway` item unless it declares another +`test_kind`. The endpoint needs the `enableGenAiReportTakeaways` feature flag. + +```json +{ + "id": "takeaway-001", + "dataset_name": "agent_document_takeaway", + "test_kind": "document_takeaway", + "question": "P01 default format, Sales page, no instructions", + "takeaway_input": { + "visualizations": ["", {"visualization_id": "", "filters": []}], + "filter_context": [], + "instructions": {"document": null, "slot": null, "format_hint": null}, + "max_characters": 300 + }, + "expected_output": { + "no_data": false, + "format": {"no_lists": false}, + "must_include": ["States one finding that connects at least two of the charts"], + "must_not_include": [], + "rubric": [] + } +} +``` + +`takeaway_input` requires `visualizations`; unknown keys are rejected. Every +`expected_output` key is optional: + +- `no_data` — whether the endpoint must answer `noData`; when either side says no data, + nothing else is graded. +- `format` — overrides of the deterministic checks, which gate pass/fail: `bold_headline`, + `max_headline_words` (10), `max_sentences` (2), `max_words` (50), `single_paragraph`, + `no_lists`, `no_headings`. `false` or `null` switches a check off; unknown keys are + rejected. `length` (`max_characters`), `no_ids` and `no_meta` always apply. +- `default_rules` — `false` drops the default judge criteria (one finding; no executive + summary, chart walk-through, background opener, figures, dates, causes the data cannot + show, chart description). +- `must_include`, `must_not_include`, `rubric` — the item's own judge criteria, graded like + `dashboard_summary`. + +`word_count` and `headline_word_count` are reported in the item detail. The default judge +criteria are calibrated with `make calibrate-takeaway` (needs `OPENAI_API_KEY`); re-run it +after changing their wording. + +The client retries 429/5xx answers and connection failures. A request error that survives +the retries is handled differently by the two entry points: `gd-eval run` aborts the item +with that error, whatever `--runs` is, while the CI entry point +(`evaluate_document_takeaway`) records it as an errored run, keeps the item's other runs, +and raises only when no run completed. A read timeout is not retried, and the CI entry +point records it as an errored run too. Any other 4xx and an unsupported URL scheme are +never retried and fail the item in both. + ### Items asked in a dashboard context (`user_context`) An item can be asked the way a user asks from an open dashboard or an attached widget. @@ -753,7 +811,8 @@ Its `user_context` is sent verbatim as `userContext` on every chat message of th follow-up and clarification messages included — the server treats a message without one as a cleared context. Requires the `enableAiContextSetup` feature flag on the target organization — without it the server ignores the dashboard view. Applies to chat items -only; `dashboard_summary` items carry their scope in `summary_input` instead. No +only; `dashboard_summary` and `document_takeaway` items carry their scope in +`summary_input` / `takeaway_input` instead. No `userContext` key is sent at all when neither the item nor, for `agentic_conversation`, the current turn sets one. @@ -806,6 +865,7 @@ from that turn on until another turn changes it, as the attached context does in | `general_question` | Text answer judged by LLM | `[llm-judge]` | | `guardrail` | Refusal/redirect (visualization response auto-fails) | `[llm-judge]` | | `dashboard_summary` | Dashboard summary (via `/summary` endpoint) scored against a rubric by LLM | `[llm-judge]` | +| `document_takeaway` | One short takeaway for a page (via `/takeaway` endpoint): deterministic format checks plus a rubric scored by LLM | `[llm-judge]` | ## Optional extras diff --git a/packages/gooddata-eval/scripts/calibrate_takeaway_judge.py b/packages/gooddata-eval/scripts/calibrate_takeaway_judge.py new file mode 100644 index 000000000..ce7630650 --- /dev/null +++ b/packages/gooddata-eval/scripts/calibrate_takeaway_judge.py @@ -0,0 +1,75 @@ +# (C) 2026 GoodData Corporation +"""Does the real judge put each calibration text on the right side of every default criterion? + +tests/test_takeaway_evaluator.py pins the wiring and the criterion wording; it cannot pin +the verdict, because the verdict is an LLM's. This does, against the texts in +tests/fixtures/takeaway_calibration.json: + +- a "pass" text must satisfy every default judge criterion on every run; +- a "fail" text must violate each criterion it names on every run. + +Run it after changing any DEFAULT_MUST_INCLUDE / DEFAULT_MUST_NOT_INCLUDE wording: + + OPENAI_API_KEY=... make calibrate-takeaway + +It is a script rather than a test because CI has no OPENAI_API_KEY. Exits 1 on any mismatch. +""" + +import json +import os +import sys +from pathlib import Path + +from gooddata_eval.core.evaluators.takeaway import ( + DEFAULT_MUST_INCLUDE, + DEFAULT_MUST_NOT_INCLUDE, + DocumentTakeawayEvaluator, +) +from gooddata_eval.core.models import ChatResult, DatasetItem, TakeawayInput + +RUNS = 3 +FIXTURE = Path(__file__).resolve().parent.parent / "tests" / "fixtures" / "takeaway_calibration.json" + +CRITERIA = {f"default_include_{k}": k for k in DEFAULT_MUST_INCLUDE} | { + f"default_exclude_{k}": k for k in DEFAULT_MUST_NOT_INCLUDE +} + + +def main() -> int: + if not os.environ.get("OPENAI_API_KEY"): + raise SystemExit("OPENAI_API_KEY required") + fixture = json.loads(FIXTURE.read_text()) + evaluator = DocumentTakeawayEvaluator() + print(f"judge model: {evaluator._positive_judge.model}, runs: {RUNS}") + mismatches = 0 + for case in fixture["texts"]: + item = DatasetItem( + id=case["label"], + dataset_name="takeaway_calibration", + test_kind="document_takeaway", + question=fixture["question"], + expected_output={}, + takeaway_input=TakeawayInput(visualizations=["calibration"]), + ) + chat = ChatResult(text_response=case["text"], raw_response={"takeaway": case["text"], "noData": False}) + satisfied = dict.fromkeys(CRITERIA.values(), 0) + for _ in range(RUNS): + detail = evaluator.evaluate(item, chat).detail + for key, name in CRITERIA.items(): + satisfied[name] += detail.get(key) is True + want_pass = case["want"] == "pass" + must = dict.fromkeys(satisfied, RUNS) if want_pass else dict.fromkeys(case["criteria"], 0) + print(f"\n{case['label']} (want {case['want']})") + for name, count in satisfied.items(): + flag = "" + if name in must: + ok = count == must[name] + mismatches += not ok + flag = "OK" if ok else "MISMATCH" + print(f" {name:20} satisfied {count}/{RUNS} {flag}") + print(f"\n{'all calibration texts on the right side' if not mismatches else f'{mismatches} mismatch(es)'}") + return 1 if mismatches else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index 4b4b623d8..89a770030 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -7,7 +7,7 @@ import threading from datetime import datetime, timezone from pathlib import Path -from typing import get_args +from typing import Any, get_args import httpx from gooddata_api_client.exceptions import ApiException, ApiTypeError @@ -36,32 +36,31 @@ from gooddata_eval.core.reporting.json_report import build_multi_model_report, write_multi_model_report from gooddata_eval.core.runner import ItemReport, run_items from gooddata_eval.core.summary.http_client import SummaryClient +from gooddata_eval.core.takeaway.http_client import TakeawayClient from gooddata_eval.core.timing import TIMERS_ENV_VAR from gooddata_eval.core.workspace import ModelResolutionError, WorkspaceModelController _EXIT_OK = 0 _EXIT_OPERATIONAL_ERROR = 2 -_SUMMARY_TEST_KIND = "dashboard_summary" class _RoutingBackend: """Dispatch each item to the right backend by test_kind. - `dashboard_summary` items go to the dedicated summary endpoint; everything - else uses the conversational chat endpoint. + Kinds in `by_kind` go to their dedicated endpoint client (`dashboard_summary` to the + summary endpoint, `document_takeaway` to the takeaway endpoint); everything else uses + the conversational chat endpoint. """ - def __init__(self, chat: ChatClient, summary: SummaryClient): + def __init__(self, chat: ChatClient, by_kind: dict[str, Any]): self._chat = chat - self._summary = summary + self._by_kind = by_kind def ask(self, item: DatasetItem) -> ChatResult: - if item.test_kind == _SUMMARY_TEST_KIND: - return self._summary.ask(item) - return self._chat.ask(item) + return self._by_kind.get(item.test_kind, self._chat).ask(item) def close(self) -> None: - for backend in (self._chat, self._summary): + for backend in (self._chat, *self._by_kind.values()): if hasattr(backend, "close"): backend.close() @@ -614,7 +613,14 @@ def on_langfuse_item_done( turn_timeout_s=config.turn_timeout_s, item_timeout_s=config.item_timeout_s, ), - SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id), + { + "dashboard_summary": SummaryClient( + host=config.host, token=config.token, workspace_id=config.workspace_id + ), + "document_takeaway": TakeawayClient( + host=config.host, token=config.token, workspace_id=config.workspace_id + ), + }, ) try: single_report = run_items( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/_retry.py b/packages/gooddata-eval/src/gooddata_eval/core/_retry.py new file mode 100644 index 000000000..982f8cb74 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/_retry.py @@ -0,0 +1,71 @@ +# (C) 2026 GoodData Corporation +"""Bounded exponential-backoff retry shared by the gen-ai HTTP clients. + +The budget is read from the ``GOODDATA_EVAL_CHAT_*`` environment on every call, so the chat +and takeaway clients retune together. +""" + +from __future__ import annotations + +import logging +import os +import time +from collections.abc import Callable +from typing import TypeVar + +_log = logging.getLogger(__name__) + +T = TypeVar("T") + +# 500 is here on evidence, not on principle: in one visualization eval batch it hard-failed +# 10 of 56 runs with zero retry attempts, and every affected question scored normally when the +# same question/model ran again the next day -- i.e. transient gen-ai faults, not deterministic +# server bugs. A genuinely deterministic 500 still terminates, just after the bounded backoff. +RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({429, 500, 502, 503, 504}) + + +def int_env(name: str, default: int) -> int: + """Read an int from the environment, falling back to ``default`` when unset or blank.""" + raw = os.getenv(name) + return int(raw) if raw else default + + +def float_env(name: str, default: float) -> float: + """Read a float from the environment, falling back to ``default`` when unset or blank.""" + raw = os.getenv(name) + return float(raw) if raw else default + + +# Retry budget defaults, giving a ~2 min worst-case cap per send (5/10/20/40/60s). +# Each is overridable via env so CI can retune without cutting a new release -- +# read per call rather than at import, so an exported value cannot silently +# rewrite what a test that patches these attributes expects. +_MAX_RETRIES_DEFAULT = 5 +_INITIAL_BACKOFF_S_DEFAULT = 5.0 +_BACKOFF_FACTOR_DEFAULT = 2.0 +_MAX_BACKOFF_S_DEFAULT = 60.0 + + +def retry_transient(operation: Callable[[], T], *, is_retryable: Callable[[Exception], bool]) -> T: + """Run ``operation``; retry retryable failures with bounded exponential backoff.""" + max_retries = int_env("GOODDATA_EVAL_CHAT_MAX_RETRIES", _MAX_RETRIES_DEFAULT) + delay = float_env("GOODDATA_EVAL_CHAT_INITIAL_BACKOFF_S", _INITIAL_BACKOFF_S_DEFAULT) + factor = float_env("GOODDATA_EVAL_CHAT_BACKOFF_FACTOR", _BACKOFF_FACTOR_DEFAULT) + max_backoff = float_env("GOODDATA_EVAL_CHAT_MAX_BACKOFF_S", _MAX_BACKOFF_S_DEFAULT) + for attempt in range(max_retries + 1): # 0..N => N retries + 1 initial attempt + try: + return operation() + except Exception as exc: # noqa: PERF203 — retry loop: per-attempt try/except is intentional + if attempt == max_retries or not is_retryable(exc): + raise + sleep_s = min(delay, max_backoff) + _log.warning( + "Transient gen-ai error (attempt %d/%d): %s; retrying in %.0fs", + attempt + 1, + max_retries + 1, + exc, + sleep_s, + ) + time.sleep(sleep_s) + delay *= factor + raise AssertionError("unreachable") # loop either returns or raises diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index ad58382e0..f5e77b2d0 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -20,11 +20,12 @@ import re import time from dataclasses import dataclass, field -from typing import Any, Callable, Iterable, TypeVar +from typing import Any, Iterable from urllib.parse import quote import httpx +from gooddata_eval.core._retry import RETRYABLE_STATUS_CODES, float_env, int_env, retry_transient from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort from gooddata_eval.core.langfuse.item_scope import JoinedRun, baggage_entries, current_scope, mark_joined, trace_ids_for from gooddata_eval.core.models import ChatResult, DatasetItem @@ -32,6 +33,13 @@ _log = logging.getLogger(__name__) +# The retry helpers live in core/_retry.py, shared with the takeaway client. The private +# names stay importable from here: gdc-nas tavern-e2e imports them from this module. +_int_env = int_env +_float_env = float_env +_RETRYABLE_STATUS_CODES = RETRYABLE_STATUS_CODES +_retry_transient = retry_transient + SSE_DATA_PREFIX = "data: " SSE_EVENT_PREFIX = "event: " # gen-ai's last event, only if at least one item was already emitted (conversations_controller.py). @@ -39,11 +47,6 @@ # gen-ai's first event of a turn; its data carries the turn's responseId and traceId. _RESPONSE_STARTED_EVENT = "response_started" -# 500 is here on evidence, not on principle: in one visualization eval batch it hard-failed -# 10 of 56 runs with zero retry attempts, and every affected question scored normally when the -# same question/model ran again the next day -- i.e. transient gen-ai faults, not deterministic -# server bugs. A genuinely deterministic 500 still terminates, just after the bounded backoff. -_RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({429, 500, 502, 503, 504}) _METADATA_SYNC_MARKER = "METADATA_SYNC_IN_PROGRESS" # `reason` values gen-ai puts on the error event of a turn that finished without an answer. _TURN_INCOMPLETE_REASONS: frozenset[str] = frozenset({"max_iterations", "max_tokens", "content_filter"}) @@ -112,27 +115,6 @@ class TurnTimeoutError(ChatError): retrying it spends the budget again.""" -def _int_env(name: str, default: int) -> int: - """Read an int from the environment, falling back to ``default`` when unset or blank.""" - raw = os.getenv(name) - return int(raw) if raw else default - - -def _float_env(name: str, default: float) -> float: - """Read a float from the environment, falling back to ``default`` when unset or blank.""" - raw = os.getenv(name) - return float(raw) if raw else default - - -# Retry budget defaults, giving a ~2 min worst-case cap per send (5/10/20/40/60s). -# Each is overridable via env so CI can retune without cutting a new release -- -# read per call rather than at import, so an exported value cannot silently -# rewrite what a test that patches these attributes expects. -_MAX_RETRIES_DEFAULT = 5 -_INITIAL_BACKOFF_S_DEFAULT = 5.0 -_BACKOFF_FACTOR_DEFAULT = 2.0 -_MAX_BACKOFF_S_DEFAULT = 60.0 - # Wall-clock cap on a single agent turn, 0 = uncapped. httpx's `timeout` is per-read, # so an agent that keeps emitting reasoning events can stream for many minutes without # ever tripping it -- this is what bounds a runaway item and lets the run move on. @@ -199,14 +181,11 @@ def set_default_item_timeout(seconds: float | None) -> None: _ITEM_TIMEOUT_S = seconds or 0.0 -T = TypeVar("T") - - def _is_retryable_exc(exc: Exception) -> bool: if isinstance(exc, TransientChatError): return True if isinstance(exc, httpx.HTTPStatusError): - return exc.response.status_code in _RETRYABLE_STATUS_CODES + return exc.response.status_code in RETRYABLE_STATUS_CODES # Mid-stream disconnect ("peer closed connection without sending complete # message body") -- pure network flake, not a real agent/content failure. # Confirmed live: contaminated ~1-4% of visualization runs with a hard @@ -214,31 +193,6 @@ def _is_retryable_exc(exc: Exception) -> bool: return isinstance(exc, httpx.RemoteProtocolError) -def _retry_transient(operation: Callable[[], T], *, is_retryable: Callable[[Exception], bool]) -> T: - """Run ``operation``; retry retryable failures with bounded exponential backoff.""" - max_retries = _int_env("GOODDATA_EVAL_CHAT_MAX_RETRIES", _MAX_RETRIES_DEFAULT) - delay = _float_env("GOODDATA_EVAL_CHAT_INITIAL_BACKOFF_S", _INITIAL_BACKOFF_S_DEFAULT) - factor = _float_env("GOODDATA_EVAL_CHAT_BACKOFF_FACTOR", _BACKOFF_FACTOR_DEFAULT) - max_backoff = _float_env("GOODDATA_EVAL_CHAT_MAX_BACKOFF_S", _MAX_BACKOFF_S_DEFAULT) - for attempt in range(max_retries + 1): # 0..N => N retries + 1 initial attempt - try: - return operation() - except Exception as exc: # noqa: PERF203 — retry loop: per-attempt try/except is intentional - if attempt == max_retries or not is_retryable(exc): - raise - sleep_s = min(delay, max_backoff) - _log.warning( - "Transient gen-ai error (attempt %d/%d): %s; retrying in %.0fs", - attempt + 1, - max_retries + 1, - exc, - sleep_s, - ) - time.sleep(sleep_s) - delay *= factor - raise AssertionError("unreachable") # loop either returns or raises - - @dataclass class _SseAccumulator: text_parts: list[str] = field(default_factory=list) @@ -456,7 +410,7 @@ def parse_sse_lines(lines: Iterable[str]) -> ChatResult: raise TurnIncompleteError( message, status_code=code, detail=detail, partial_result=_build_chat_result(acc), reason=reason ) - if code in _RETRYABLE_STATUS_CODES: + if code in RETRYABLE_STATUS_CODES: raise TransientChatError( message, status_code=code, detail=detail, partial_result=_build_chat_result(acc), reason=reason ) @@ -557,7 +511,7 @@ def _do() -> str: # NOTE: retrying create is not idempotent — a created-then-503 can leak an # orphaned (ephemeral) conversation. Acceptable for eval; do not reuse blindly. - conversation_id = _retry_transient(_do, is_retryable=_is_retryable_exc) + conversation_id = retry_transient(_do, is_retryable=_is_retryable_exc) # Anchor the per-item clock here: for an agentic item this conversation carries # every turn, so the budget must run from its creation, not from each send. self._conversation_started = time.monotonic() @@ -603,7 +557,7 @@ def _do() -> ChatResult: return result try: - result = _retry_transient(_do, is_retryable=_is_retryable_exc) + result = retry_transient(_do, is_retryable=_is_retryable_exc) except ChatError as exc: if joined is not None and exc.partial_result is not None: self._record_join(conversation_id, exc.partial_result.trace_id, joined) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py index 415cc77f2..0c9047460 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py @@ -15,9 +15,10 @@ from typing import Any, TypeVar, cast import httpx +from pydantic import ValidationError from gooddata_eval.core.langfuse._env import make_http_client -from gooddata_eval.core.models import DatasetItem, SummaryInput +from gooddata_eval.core.models import DatasetItem, SummaryInput, TakeawayInput _PAGE_SIZE = 100 @@ -59,6 +60,27 @@ def _summary_input_from_raw(raw: dict, expected_output: Any) -> SummaryInput | N return SummaryInput.model_validate(candidate) if candidate is not None else None +def _takeaway_input_from_raw(raw: dict, expected_output: Any) -> TakeawayInput | None: + """Locate a document_takeaway item's `takeaway_input`. + + A value that is not an object raises rather than being skipped: skipping it would + drop the item back to the CLI's --kind and judge it as something it was not written as. + """ + for source in (raw.get("input"), raw.get("metadata"), expected_output): + if not isinstance(source, dict) or source.get("takeaway_input") is None: + continue + found = source["takeaway_input"] + if not isinstance(found, dict): + raise ValueError( + f"Langfuse item {raw.get('id')!r}: takeaway_input must be a JSON object, got {type(found).__name__}" + ) + try: + return TakeawayInput.model_validate(found) + except ValidationError as e: + raise ValueError(f"Langfuse item {raw.get('id')!r}: invalid takeaway_input: {e}") from e + return None + + def _user_context_from_raw(raw: dict) -> dict[str, Any] | None: """Locate an item's `user_context` -- relayed verbatim as the chat request's `userContext`. @@ -115,7 +137,11 @@ def _item_from_raw(raw: dict, *, dataset_name: str, test_kind: str) -> DatasetIt """Map a Langfuse REST API dataset-item dict to a DatasetItem.""" # REST API returns camelCase: expectedOutput, not expected_output expected_output = raw.get("expectedOutput") or raw.get("expected_output") - resolved_kind = _infer_test_kind(expected_output, test_kind, raw.get("metadata")) + takeaway_input = _takeaway_input_from_raw(raw, expected_output) + # An item carrying `takeaway_input` can only be a takeaway item, so it stands in for + # the CLI default; an explicit test_kind or a structural match still wins. + default_kind = "document_takeaway" if takeaway_input is not None else test_kind + resolved_kind = _infer_test_kind(expected_output, default_kind, raw.get("metadata")) return DatasetItem( id=str(raw["id"]), dataset_name=raw.get("datasetName") or dataset_name, @@ -123,6 +149,7 @@ def _item_from_raw(raw: dict, *, dataset_name: str, test_kind: str) -> DatasetIt question=_question_from_input(raw.get("input")), expected_output=expected_output, summary_input=_summary_input_from_raw(raw, expected_output), + takeaway_input=takeaway_input, user_context=_user_context_from_raw(raw), ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py index 30b154821..16a9e91b7 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py @@ -20,7 +20,8 @@ ) } -# LLM-judge evaluators (general_question, guardrail, dashboard_summary, knowledge_question) +# LLM-judge evaluators (general_question, guardrail, dashboard_summary, knowledge_question, +# document_takeaway) # require the [llm-judge] extra. Their modules are imported lazily on first use so the CLI # starts without openai. # @@ -34,12 +35,14 @@ "guardrail": "gooddata_eval.core.evaluators.guardrail", "dashboard_summary": "gooddata_eval.core.evaluators.summary", "knowledge_question": "gooddata_eval.core.evaluators.general_question", + "document_takeaway": "gooddata_eval.core.evaluators.takeaway", } _LAZY_EVALUATOR_CLASSES: dict[str, str] = { "general_question": "GeneralQuestionEvaluator", "guardrail": "GuardrailEvaluator", "dashboard_summary": "DashboardSummaryEvaluator", "knowledge_question": "GeneralQuestionEvaluator", + "document_takeaway": "DocumentTakeawayEvaluator", } @@ -66,7 +69,7 @@ def supported_test_kinds() -> set[str]: """Return all supported test_kind values. LLM-judge kinds (general_question, guardrail, dashboard_summary, - knowledge_question) are excluded when the [llm-judge] extra (openai) is + knowledge_question, document_takeaway) are excluded when the [llm-judge] extra (openai) is not installed — those items are skipped rather than erroring out mid-run. """ kinds = set(_EAGER_EVALUATORS) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_rubric.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_rubric.py new file mode 100644 index 000000000..26a6c3095 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_rubric.py @@ -0,0 +1,94 @@ +# (C) 2026 GoodData Corporation +"""Per-criterion LLM-judge grading shared by the rubric-scored single-shot evaluators.""" + +from collections.abc import Sequence +from typing import Any + +from gooddata_eval.core.evaluators._llm_judge import LLMJudge, score_run +from gooddata_eval.core.evaluators.base import ItemEvaluation + +# A criterion keyed by the name its verdict is stored under in `detail`. +Criterion = tuple[str, str] + + +def grade_rubric( + positive_judge: LLMJudge, + violation_judge: LLMJudge, + *, + question: str, + actual: str, + must_include: Sequence[Criterion], + must_not_include: Sequence[Criterion], + rubric: Sequence[Criterion], + detail: dict[str, Any], + passed: bool = True, +) -> ItemEvaluation: + """Grade every criterion with one judge request each and decide the item. + + `must_include` and `must_not_include` gate the verdict, `rubric` only informs quality. + `must_not_include` criteria are put to `violation_judge` as a presence question and + inverted here. `detail` may already hold verdicts the caller decided without a judge, + with `passed` carrying their outcome; every bool in it counts towards quality. + + A criterion the judge returns nothing readable for is *ungraded*: recorded without a + bool, so it neither fails the item nor enters the quality denominator, but it cannot + carry a pass either. When nothing that decides the item was graded, the run has no + verdict and comes back with `error` set. + """ + # One judge request PER CRITERION, so an unreadable response has to be confined to + # its own criterion: letting it raise would discard every criterion already graded + # and abandon the ones after it, losing a 7-criterion item to one bad body. + decided_without_judge = passed + ungraded = 0 + graded_gating = 0 + graded_rubric = 0 + + def _grade(judge: LLMJudge, criterion: str, key: str, *, invert: bool = False) -> bool | None: + nonlocal ungraded + verdict = score_run(judge, input=question, expected_output=criterion, actual_output=actual) + if verdict.error is not None: + ungraded += 1 + detail[f"{key}_reason"] = f"UNGRADED: {verdict.error}" + return None + # invert: the violation judge answers "is the characteristic present?", so the + # criterion is satisfied exactly when it says no. + ok = not verdict.passed if invert else verdict.passed + detail[key] = ok + detail[f"{key}_reason"] = verdict.reasoning + return ok + + # `is True`, not `is not False`: a gating criterion the judge could not grade is not + # a failure, but it cannot carry a pass either. + for judge, criteria, invert in ((positive_judge, must_include, False), (violation_judge, must_not_include, True)): + for key, criterion in criteria: + ok = _grade(judge, criterion, key, invert=invert) + graded_gating += ok is not None + passed = passed and ok is True + + for key, criterion in rubric: + graded_rubric += _grade(positive_judge, criterion, key) is not None + + if ungraded: + detail["ungraded_criteria"] = ungraded + bool_checks = [v for v in detail.values() if isinstance(v, bool)] + quality = sum(1 for v in bool_checks if v) / len(bool_checks) if bool_checks else 0.0 + + # Keyed on the criteria that decide the verdict, not on any graded bool: rubric + # criteria never gate `passed`, so one graded rubric line must not certify a pass + # whose every mandatory fact went unassessed. A rubric-only item has no gating + # criteria, so there any graded line is a verdict. A caller-decided failure already + # is a verdict. + gating = len(must_include) + len(must_not_include) + judge_verdict = graded_gating if gating else graded_rubric + if ungraded and decided_without_judge and not judge_verdict: + return ItemEvaluation( + passed=False, + rank_key=(-1, quality), + detail=detail, + error=( + f"judge returned no readable verdict for any of the {gating or ungraded} criterion(s) " + "that decide this item." + ), + ) + + return ItemEvaluation(passed=passed, rank_key=(int(passed), quality), detail=detail) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/summary.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/summary.py index 0c680b0c8..46fb9e27e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/summary.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/summary.py @@ -27,7 +27,8 @@ from typing import Any -from gooddata_eval.core.evaluators._llm_judge import LLMJudge, score_run +from gooddata_eval.core.evaluators._llm_judge import LLMJudge +from gooddata_eval.core.evaluators._rubric import grade_rubric from gooddata_eval.core.evaluators._text_utils import extract_text from gooddata_eval.core.evaluators.base import ItemEvaluation from gooddata_eval.core.models import ChatResult, DatasetItem @@ -74,66 +75,13 @@ def _criteria(expected_output: Any) -> tuple[list[str], list[str], list[str]]: def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation: actual = extract_text(chat_result) must_include, must_not_include, rubric = self._criteria(item.expected_output) - - detail: dict[str, Any] = {"actual_output": actual} - passed = True - - # One judge request PER CRITERION, so an unreadable response has to be confined to - # its own criterion: letting it raise would discard every criterion already graded - # and abandon the ones after it, losing a 7-criterion item to one bad body. An - # ungraded criterion is recorded but stored without a bool, which keeps it out of - # the failing list and the quality denominator below: counting it as failed would - # invent a score the judge never gave. - ungraded = 0 - - def _grade(judge: LLMJudge, criterion: str, key: str, *, invert: bool = False) -> bool | None: - nonlocal ungraded - verdict = score_run(judge, input=item.question, expected_output=criterion, actual_output=actual) - if verdict.error is not None: - ungraded += 1 - detail[f"{key}_reason"] = f"UNGRADED: {verdict.error}" - return None - # invert: the violation judge answers "is the characteristic present?", so the - # criterion is satisfied exactly when it says no. - ok = not verdict.passed if invert else verdict.passed - detail[key] = ok - detail[f"{key}_reason"] = verdict.reasoning - return ok - - # `is True`, not `is not False`: a gating criterion the judge could not grade is not - # a failure, but it cannot carry a pass either. - for i, criterion in enumerate(must_include): - ok = _grade(self._positive_judge, criterion, f"include_{i}") - passed = passed and ok is True - - for i, criterion in enumerate(must_not_include): - ok = _grade(self._violation_judge, criterion, f"exclude_{i}", invert=True) - passed = passed and ok is True - - for i, criterion in enumerate(rubric): - # Rubric criteria inform quality but never gate `passed`. - _grade(self._positive_judge, criterion, f"rubric_{i}") - - if ungraded: - detail["ungraded_criteria"] = ungraded - bool_checks = [v for v in detail.values() if isinstance(v, bool)] - quality = sum(1 for v in bool_checks if v) / len(bool_checks) if bool_checks else 0.0 - - # Keyed on the criteria that decide the verdict, not on any graded bool: rubric - # criteria never gate `passed`, so one graded rubric line must not certify a pass - # whose every mandatory fact went unassessed. A rubric-only item has no gating - # criteria, so there any graded line is a verdict. - gating = len(must_include) + len(must_not_include) - graded_gating = [v for k, v in detail.items() if isinstance(v, bool) and not k.startswith("rubric_")] - if ungraded and not (graded_gating if gating else bool_checks): - return ItemEvaluation( - passed=False, - rank_key=(-1, quality), - detail=detail, - error=( - f"judge returned no readable verdict for any of the {gating or ungraded} criterion(s) " - "that decide this item." - ), - ) - - return ItemEvaluation(passed=passed, rank_key=(int(passed), quality), detail=detail) + return grade_rubric( + self._positive_judge, + self._violation_judge, + question=item.question, + actual=actual, + must_include=[(f"include_{i}", c) for i, c in enumerate(must_include)], + must_not_include=[(f"exclude_{i}", c) for i, c in enumerate(must_not_include)], + rubric=[(f"rubric_{i}", c) for i, c in enumerate(rubric)], + detail={"actual_output": actual}, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/takeaway.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/takeaway.py new file mode 100644 index 000000000..494910938 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/takeaway.py @@ -0,0 +1,245 @@ +# (C) 2026 GoodData Corporation +"""Evaluator for document_takeaway: deterministic format checks plus an LLM-judge rubric. + +A takeaway is one finding for a page, readable in about five seconds: a bold headline, +at most two short sentences, one paragraph, ≤ 50 words whatever ``maxCharacters`` allows. +`expected_output` is: + + { + "no_data": false, + "format": {...overrides of _DEFAULT_FORMAT...}, + "default_rules": true, + "must_include": ["...criteria the takeaway must satisfy..."], + "must_not_include": ["...characteristics the takeaway must not show..."], + "rubric": ["...soft quality dimensions..."] + } + +Every key is optional. The format checks need no judge and gate the verdict. A ``format`` +value of ``false`` or ``null`` switches its check off, for an item whose instruction asks +for another shape (a list, no headline). Unless ``default_rules`` is false, the +DEFAULT_MUST_INCLUDE / DEFAULT_MUST_NOT_INCLUDE judge criteria are added to the item's +own, under ``default_`` keys. Judge criteria grade like ``dashboard_summary``. + +``no_data`` decides the item alone: the endpoint's ``noData`` must equal it, and a +``noData`` answer carries no takeaway, so nothing else is graded. An expected takeaway +that came back as ``noData`` is a failure, not an empty text to judge. +""" + +import re +from typing import Any + +from gooddata_eval.core.evaluators._llm_judge import LLMJudge +from gooddata_eval.core.evaluators._rubric import Criterion, grade_rubric +from gooddata_eval.core.evaluators.base import ItemEvaluation +from gooddata_eval.core.models import ChatResult, DatasetItem + +# The judge criteria below are calibrated against tests/fixtures/takeaway_calibration.json +# with `make calibrate-takeaway`; re-run it after changing any of their wording. +DEFAULT_MUST_INCLUDE: dict[str, str] = { + "one_finding": "A reader gets one clear main finding from the first line alone.", +} + +DEFAULT_MUST_NOT_INCLUDE: dict[str, str] = { + "executive_summary": ( + "Reads as an executive summary or report: it contains section labels such as 'Summary', " + "'Key takeaways' or 'Things to watch', or lists several separate findings one after another, " + "or gives recommendations, an outlook or a closing conclusion." + ), + "chart_walkthrough": "Walks through the charts one by one instead of stating one finding that connects them.", + "background_opener": "Opens with background or a description of the page instead of the finding itself.", + "exact_figure": ( + "States a figure in digits: an amount, count, percentage or share, including rounded or hedged ones " + "such as 'about $270M', 'roughly 1.2K', 'in the low $100Ks' or '~40%'. A rough size or direction in words, such as " + "'roughly doubled', 'about a third' or 'most orders', does not count. Numbers that are part of a " + "visualization or metric title, such as 'Top 10', do not count." + ), + "date_or_period": ( + "Names a specific date, month, quarter, year or named period. Relative references such as " + "'this period' do not count." + ), + "unshown_cause": ( + "Makes an explicit causal claim (such as 'because', 'due to' or 'driven by') that attributes a change " + "to a factor that is not in the data, such as a campaign, a season or a market event. Comparisons " + "between segments, and naming the segment in the data that led a change (such as 'led by the " + "Nordics'), are not causal claims." + ), + "chart_description": "Describes what a chart displays instead of what the data means.", +} + +_DEFAULT_FORMAT: dict[str, Any] = { + "bold_headline": True, + "max_headline_words": 10, + "max_sentences": 2, + "max_words": 50, + "single_paragraph": True, + "no_lists": True, + "no_headings": True, +} + +_POSITIVE_STEPS = [ + "Read the INPUT (a label for the page the takeaway was written for) and the EXPECTED OUTPUT " + "(one criterion the takeaway must satisfy).", + "Read the ACTUAL OUTPUT (the generated takeaway).", + "Score 1 if the actual output clearly satisfies the criterion.", + "Score 0 if the criterion is missing, contradicted, or only partially addressed.", +] + +# A presence question, inverted in code -- see the same split in summary.py. +_VIOLATION_STEPS = [ + "Read the CHARACTERISTIC described in EXPECTED OUTPUT, including any exceptions it states.", + "Read the ACTUAL OUTPUT (the generated takeaway).", + "Score 1 if the actual output clearly exhibits the described characteristic.", + "Score 0 if it does not exhibit it.", +] + +_HEADLINE = re.compile(r"^\s*\*\*(.+?)\*\*", re.DOTALL) +# A terminator, any closing quotes or brackets, then whitespace before the next sentence. +_SENTENCE_BREAK = re.compile(r"[.!?]+[\"'”’»)\]]*\s+(?=\S)") +_SENTENCE_OPENERS = "\"'“‘„«([" +_ABBREVIATIONS = frozenset({"vs", "e.g", "i.e", "u.s", "cca", "např", "tj", "tzn", "apod", "resp", "atd"}) +_BLANK_LINE = re.compile(r"\n\s*\n") +_HEADING_LINE = re.compile(r"^\s*#{1,6}\s", re.MULTILINE) +_LIST_LINE = re.compile(r"^\s*(?:[-*•]|\d+[.)])\s", re.MULTILINE) +_IDS = re.compile( + r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|\{\w+/[^}]*\}|\blabel\.\w", + re.IGNORECASE, +) +_META_OPENER = re.compile( + r"^(?:summary|key takeaways?|based on the data|visualization metadata|this chart shows)\b", + re.IGNORECASE, +) + + +def _words(text: str) -> int: + return sum(1 for token in text.replace("**", " ").split() if any(ch.isalnum() for ch in token)) + + +def _limit_on(value: Any) -> bool: + return value is not None and value is not False + + +def _sentences(text: str) -> int: + """Sentences in `text`; a break needs the next sentence to open with a capital, digit or quote. + + That keeps "vs. last", "U.S. sales" or "např. v" from splitting one sentence in two; a + terminator right after a known abbreviation ("vs. Europe", "e.g. Nordics") is no break. + """ + plain = text.replace("**", "").strip() + if not plain: + return 0 + breaks = 0 + for match in _SENTENCE_BREAK.finditer(plain): + nxt = plain[match.end()] + token = (plain[: match.start()].rsplit(maxsplit=1) or [""])[-1].lstrip("\"'“‘„«([").lower() + if token in _ABBREVIATIONS: + continue + breaks += nxt.isupper() or nxt.isdigit() or nxt in _SENTENCE_OPENERS + return breaks + 1 + + +def _has_meta_opener(text: str, headline: str | None, body: str) -> bool: + openers = [line.strip().lstrip("#*").strip() for line in text.splitlines()] + openers += [headline or "", body.strip()] + return any(_META_OPENER.match(opener) for opener in openers) + + +class DocumentTakeawayEvaluator: + test_kind = "document_takeaway" + + def __init__(self): + self._positive_judge = LLMJudge(evaluation_steps=_POSITIVE_STEPS) + self._violation_judge = LLMJudge(evaluation_steps=_VIOLATION_STEPS) + + @staticmethod + def validate(item: DatasetItem) -> None: + """Reject an item whose `expected_output` this evaluator cannot read, before any request. + + Raises: + ValueError: `format` is not an object, carries a key the checks do not know, or a criteria key + is not a JSON array. + """ + expected = item.expected_output if isinstance(item.expected_output, dict) else {} + overrides = expected.get("format") or {} + if not isinstance(overrides, dict): + raise ValueError(f"document_takeaway item '{item.id}': format must be a JSON object") + unknown = sorted(set(overrides) - set(_DEFAULT_FORMAT)) + if unknown: + raise ValueError( + f"document_takeaway item '{item.id}': unknown format key(s) {unknown}; known: {sorted(_DEFAULT_FORMAT)}" + ) + for key in ("must_include", "must_not_include", "rubric"): + if not isinstance(expected.get(key, []), list): + raise ValueError(f"document_takeaway item '{item.id}': {key} must be a JSON array") + + def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation: + self.validate(item) + expected: dict[str, Any] = item.expected_output if isinstance(item.expected_output, dict) else {} + raw = chat_result.raw_response or {} + returned_no_data = raw.get("noData") is True + text = chat_result.text_response or "" + + expected_no_data = expected.get("no_data") is True + if expected_no_data or returned_no_data: + ok = expected_no_data and returned_no_data and raw.get("takeaway") is None + detail: dict[str, Any] = {"actual_output": text, "no_data": ok} + return ItemEvaluation(passed=ok, rank_key=(int(ok), float(ok)), detail=detail) + + detail = {"actual_output": text} + detail.update(self._format_checks(item, expected, text)) + passed = all(v for v in detail.values() if isinstance(v, bool)) + + must_include: list[Criterion] = [] + must_not_include: list[Criterion] = [] + if expected.get("default_rules", True) is not False: + must_include += [(f"default_include_{k}", c) for k, c in DEFAULT_MUST_INCLUDE.items()] + must_not_include += [(f"default_exclude_{k}", c) for k, c in DEFAULT_MUST_NOT_INCLUDE.items()] + must_include += [(f"include_{i}", str(c)) for i, c in enumerate(expected.get("must_include", []))] + must_not_include += [(f"exclude_{i}", str(c)) for i, c in enumerate(expected.get("must_not_include", []))] + rubric = [(f"rubric_{i}", str(c)) for i, c in enumerate(expected.get("rubric", []))] + + return grade_rubric( + self._positive_judge, + self._violation_judge, + question=item.question, + actual=text, + must_include=must_include, + must_not_include=must_not_include, + rubric=rubric, + detail=detail, + passed=passed, + ) + + @staticmethod + def _format_checks(item: DatasetItem, expected: dict[str, Any], text: str) -> dict[str, Any]: + """Deterministic checks keyed by name, plus `word_count` and `headline_word_count`.""" + fmt = {**_DEFAULT_FORMAT, **(expected.get("format") or {})} + checks: dict[str, Any] = {} + + match = _HEADLINE.match(text) if fmt["bold_headline"] else None + headline = match.group(1).strip() if match else None + body = text[match.end() :] if match else text + word_count = _words(text) + headline_word_count = _words(headline) if headline else 0 + checks["word_count"] = word_count + checks["headline_word_count"] = headline_word_count + + if _limit_on(fmt["max_words"]): + checks["word_budget"] = word_count <= fmt["max_words"] + if fmt["bold_headline"]: + checks["bold_headline"] = headline is not None + if _limit_on(fmt["max_headline_words"]): + checks["headline_words"] = headline_word_count <= fmt["max_headline_words"] + if _limit_on(fmt["max_sentences"]): + checks["max_sentences"] = _sentences(body) <= fmt["max_sentences"] + if fmt["single_paragraph"]: + checks["single_paragraph"] = not _BLANK_LINE.search(text.strip()) and text.strip().count("\n") <= 1 + max_characters = item.takeaway_input.max_characters if item.takeaway_input else None + if max_characters is not None: + checks["length"] = len(text) <= max_characters + if fmt["no_headings"]: + checks["no_headings"] = not _HEADING_LINE.search(text) + if fmt["no_lists"]: + checks["no_lists"] = not _LIST_LINE.search(text) + checks["no_ids"] = not _IDS.search(text) + checks["no_meta"] = not _has_meta_opener(text, headline, body) + return checks diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index 5bcdd0045..d71f34d93 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -347,6 +347,9 @@ class ChatResult(BaseModel): stream_ended: bool = False # Wall-clock seconds for the whole chat turn, timed by the client. turn_wall_clock_sec: float | None = None + # The endpoint's whole JSON response, for single-shot endpoints whose answer carries + # more than its text (the takeaway endpoint's `noData`). None for chat responses. + raw_response: dict[str, Any] | None = None def shift_and_index_events( @@ -467,6 +470,42 @@ class SummaryInput(BaseModel): format_hint: str | None = None +class TakeawayVisualization(BaseModel): + """A takeaway visualization placed with its own filters.""" + + model_config = ConfigDict(extra="forbid") + + visualization_id: str + filters: list[dict] = Field(default_factory=list) + + +class TakeawayInstructions(BaseModel): + model_config = ConfigDict(extra="forbid") + + document: str | None = None + slot: str | None = None + format_hint: str | None = None + + +class TakeawayInput(BaseModel): + """Structured input for the `document_takeaway` test kind. + + Maps onto the takeaway endpoint's request body + (`POST /api/v1/ai/workspaces/{ws}/takeaway`). Authored in snake_case in the + dataset; the TakeawayClient maps it to the endpoint's camelCase fields. Unknown + keys are rejected at every level so a typo in a dataset item fails loudly instead + of being dropped. + """ + + model_config = ConfigDict(extra="forbid") + + visualizations: list[str | TakeawayVisualization] = Field(min_length=1) + filter_context: list[dict] = Field(default_factory=list) + instructions: TakeawayInstructions = Field(default_factory=TakeawayInstructions) + max_characters: int | None = None + agent_id: str | None = None + + class DatasetItem(BaseModel): """Common dataset envelope. `expected_output` stays raw; each evaluator parses its own shape.""" @@ -479,6 +518,8 @@ class DatasetItem(BaseModel): expected_output: Any # Only used by the `dashboard_summary` test kind; ignored by all others. summary_input: SummaryInput | None = None + # Only used by the `document_takeaway` test kind; ignored by all others. + takeaway_input: TakeawayInput | None = None # Relayed verbatim as the chat request's `userContext`. Deliberately opaque: gen-ai owns # that schema (a discriminated union of view/widget descriptors), so re-modelling it here # would only create a second copy to keep in sync. diff --git a/packages/gooddata-eval/src/gooddata_eval/core/takeaway/__init__.py b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/__init__.py new file mode 100644 index 000000000..efe7c60c8 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/__init__.py @@ -0,0 +1 @@ +# (C) 2026 GoodData Corporation diff --git a/packages/gooddata-eval/src/gooddata_eval/core/takeaway/evaluate.py b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/evaluate.py new file mode 100644 index 000000000..1ffa3e934 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/evaluate.py @@ -0,0 +1,275 @@ +# (C) 2026 GoodData Corporation +"""Entry point for evaluating one document_takeaway item K times, for the Tavern shim. + +The CLI evaluates takeaway items through the single-shot runner; CI goes through +``evaluate_document_takeaway``, which owns the K-run loop, the gate and the Langfuse +experiment item the same way the ``evaluate_agentic_*`` entry points do, so the nightly +report reads it unchanged. + +The takeaway endpoint returns no conversation id, so there is no gen-ai trace to link: +every score is written to gd-eval's own experiment span. + +A run whose request still fails transiently after the client's retries (5xx, 429, a +connection failure or a read timeout) is recorded as an errored run and left ungraded, +like a run the judge could not read; the item raises it only when no run completed. Any +other 4xx (flag off, no agent, invalid body) or an unsupported URL scheme is a setup +problem and raises on the first run. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import Any + +import httpx + +from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) +from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError +from gooddata_eval.core.evaluators.base import ItemEvaluation +from gooddata_eval.core.evaluators.takeaway import DocumentTakeawayEvaluator +from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, ChatResult, DatasetItem +from gooddata_eval.core.takeaway.http_client import TakeawayClient +from gooddata_eval.core.timing import PhaseTimings, sum_timings + +_TEST_KIND = "document_takeaway" +_COUNT_SCORES = ("word_count", "headline_word_count") + + +class TakeawayAssertionError(AgenticAssertionError): + """Raised when a document-takeaway evaluation fails its gate.""" + + +@dataclass +class _TakeawayRun: + chat_result: ChatResult + evaluation: ItemEvaluation + latency_s: float + timings: PhaseTimings + request_error: httpx.HTTPError | None = None + + @property + def judge_error(self) -> str | None: + return self.evaluation.error + + @property + def checks(self) -> dict[str, bool]: + return {k: v for k, v in self.evaluation.detail.items() if isinstance(v, bool)} + + +def _dataset_item(dataset_item: dict, dataset_name: str) -> DatasetItem: + item = DatasetItem.model_validate( + {"id": "", "test_kind": _TEST_KIND, "expected_output": {}, **dataset_item, "dataset_name": dataset_name} + ) + if item.takeaway_input is None: + raise ValueError(f"document_takeaway item '{item.id}' is missing required 'takeaway_input'.") + return item + + +def _is_transient(exc: httpx.HTTPError) -> bool: + if isinstance(exc, httpx.HTTPStatusError): + status = exc.response.status_code + return status == 429 or status >= 500 + return isinstance(exc, httpx.TransportError) and not isinstance(exc, httpx.UnsupportedProtocol) + + +def _run_once(client: TakeawayClient, evaluator: DocumentTakeawayEvaluator, item: DatasetItem) -> _TakeawayRun: + started = time.monotonic() + try: + chat_result = client.ask(item) + except httpx.HTTPError as exc: + if not _is_transient(exc): + raise + elapsed = time.monotonic() - started + error = f"{type(exc).__name__}: {exc}" + evaluation = ItemEvaluation(passed=False, rank_key=(-2, 0.0), detail={"request_error": error}, error=error) + return _TakeawayRun(ChatResult(), evaluation, elapsed, PhaseTimings(agent_s=elapsed), request_error=exc) + asked = time.monotonic() + evaluation = evaluator.evaluate(item, chat_result) + judged = time.monotonic() + return _TakeawayRun( + chat_result, evaluation, asked - started, PhaseTimings(agent_s=asked - started, judge_s=judged - asked) + ) + + +def evaluate_document_takeaway( + *, + host: str, + token: str, + workspace_id: str, + dataset_item: dict, + dataset_name: str, + run_timestamp: str | None, + model_version_override: str | None = None, + reasoning_effort: ReasoningEffort | None = None, + k: int = 1, + gate: EvalGate = DEFAULT_GATE, + run_metadata_extra: dict | None = None, + langfuse: object | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, + client: TakeawayClient | None = None, + evaluator: DocumentTakeawayEvaluator | None = None, +) -> AgenticEvalOutcome: + """Request a takeaway K times, grade each, log to Langfuse, and raise on a failed gate. + + Args: + host: GoodData host URL. + token: API token. + workspace_id: Workspace the visualizations live in. + dataset_item: The item as stored: `id`, `question`, `expected_output`, `takeaway_input`. + dataset_name: Langfuse dataset name; prefixes the experiment run name. + run_timestamp: Shared run timestamp for the run name. + model_version_override: Model-version tag for the run; the workspace's active LLM + provider otherwise. + reasoning_effort: Only names the run: the takeaway endpoint has no reasoning option. + k: Runs per item. + gate: `any` (pass@K) or `power` (pass^K). + run_metadata_extra: Extra experiment-run metadata, e.g. `testing_framework`. + langfuse: Langfuse client; built from the LANGFUSE_* environment when None. + submit_trace_link: How the Langfuse block is run; inline by default. + client: Takeaway client to use instead of one built from host/token/workspace. + evaluator: Evaluator to use instead of a default one. + + Raises: + TakeawayAssertionError: the gate did not pass. + ValueError: the item has no `takeaway_input` or an `expected_output` the evaluator + cannot read, raised before any request. + httpx.HTTPError: the endpoint refused with a non-transient 4xx, or no run completed; + the message carries status and body. + JudgeResponseError: the judge returned no readable verdict for any run. + """ + item = _dataset_item(dataset_item, dataset_name) + evaluator = evaluator or DocumentTakeawayEvaluator() + evaluator.validate(item) + own_client = client is None + takeaway_client = client or TakeawayClient(host=host, token=token, workspace_id=workspace_id) + langfuse, window_start = open_trace_window(langfuse) + try: + runs = [_run_once(takeaway_client, evaluator, item) for _ in range(k)] + finally: + if own_client: + takeaway_client.close() + + completed = [r for r in runs if r.request_error is None] + if not completed: + raise runs[-1].request_error or RuntimeError("no takeaway run completed") + graded = [r for r in runs if r.judge_error is None] + pass_at_k = any(r.evaluation.passed for r in graded) + pass_power_k = all(r.evaluation.passed for r in runs) + + if langfuse is not None and item.id: + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(runs), gate=gate) + for run_idx, run in enumerate(runs): + if run.judge_error is not None: + continue + with ctx.observe(None, run_idx, output=run.chat_result.raw_response) as tid: + for name, value in run.checks.items(): + ctx.score(tid, name=name, value=float(value), data_type="BOOLEAN") + for name in _COUNT_SCORES: + if name in run.evaluation.detail: + ctx.score(tid, name=name, value=run.evaluation.detail[name], data_type="NUMERIC") + ctx.score( + tid, name="document_takeaway_pass", value=float(run.evaluation.passed), data_type="BOOLEAN" + ) + log_gate_scores(ctx, tid, gate=gate, pass_at_k=pass_at_k, pass_power_k=pass_power_k) + ctx.quality(tid, strict_checks=run.checks, latency_sec=run.latency_s, cost_usd=None) + + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=item.id, + conversation_ids=[], + window_start=window_start, + window_end=window_end, + suffix_runs=len(runs) > 1, + write_scores=_write_scores, + item_input=item.question, + ) + + item_timings = sum_timings([r.timings for r in runs]) + unscored = [r.judge_error for r in runs if r.judge_error is not None] + request_errors = [r.judge_error for r in runs if r.request_error is not None and r.judge_error is not None] + judge_errors = [r.judge_error for r in completed if r.judge_error is not None] + best = max(completed, key=lambda r: r.evaluation.rank_key) + runs_passed = sum(1 for r in runs if r.evaluation.passed) + detail: dict[str, Any] = { + **best.evaluation.detail, + **({"unscored_runs": len(unscored)} if unscored else {}), + **({"judge_errors": judge_errors} if judge_errors else {}), + **({"request_errors": request_errors} if request_errors else {}), + } + failed_runs = build_failed_runs(runs, passed=lambda r: r.evaluation.passed, detail=lambda r: r.evaluation.detail) + + def _attach(exc: TakeawayAssertionError | JudgeResponseError) -> None: + exc.reasoning_steps = [] + exc.conversation_id = "" + exc.response_id = None + exc.timings = item_timings + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = len(runs) + exc.failed_runs = failed_runs + + if not graded: + if request_errors: + causes = [ + *([f"judge returned no readable verdict for {len(judge_errors)} run(s)"] if judge_errors else []), + f"request failed for {len(request_errors)} run(s)", + ] + message = f"no run of {len(runs)} could be graded: {', '.join(causes)}: " + else: + message = f"judge returned no readable verdict for any of the {len(runs)} run(s): " + exc_judge = JudgeResponseError(message + " | ".join(unscored)) + _attach(exc_judge) + raise exc_judge + + if not gate_passed(gate, pass_at_k=pass_at_k, pass_power_k=pass_power_k): + failed = [name for name, ok in best.checks.items() if not ok] + parts = [ + "Document takeaway assertion failed.", + gate_failure_note(gate, runs_passed, len(runs), len(unscored)), + *([f"Failed checks: {', '.join(failed)}."] if failed else []), + *([f"Request errors: {' | '.join(request_errors)}"] if request_errors else []), + f"Takeaway: {best.chat_result.text_response!r}", + ] + exc = TakeawayAssertionError(" ".join(parts)) + _attach(exc) + raise exc + return AgenticEvalOutcome( + detail=detail, + timings=item_timings, + runs_passed=runs_passed, + runs_effective=len(runs), + failed_runs=failed_runs, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/takeaway/http_client.py b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/http_client.py new file mode 100644 index 000000000..d2ef8d9e5 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/http_client.py @@ -0,0 +1,104 @@ +# (C) 2026 GoodData Corporation +"""HTTP client for the AI Publisher takeaway endpoint. + +Like the dashboard-summary endpoint, it executes each visualization server-side and +returns plain synchronous JSON. The response is adapted into a ``ChatResult`` +(takeaway text -> ``text_response``); the whole response body is kept on +``raw_response`` so the evaluator can tell a ``noData`` answer from an empty takeaway. + +Endpoint (gen-ai service): + POST /api/v1/ai/workspaces/{workspace_id}/takeaway + +The request DTO rejects unknown keys and nulls are not sent, so every unset optional +field is omitted rather than sent as ``null``. +""" + +from typing import Any + +import httpx + +from gooddata_eval.core._retry import RETRYABLE_STATUS_CODES, retry_transient +from gooddata_eval.core.models import ChatResult, DatasetItem, TakeawayInput, TakeawayVisualization + +_PATH = "/api/v1/ai/workspaces/{workspace_id}/takeaway" + + +def _visualization_body(visualization: str | TakeawayVisualization) -> str | dict[str, Any]: + if isinstance(visualization, str): + return visualization + return {"visualizationId": visualization.visualization_id, "filters": visualization.filters} + + +# Failures that say nothing about the request. A read timeout is left out: the page was +# being computed, and retrying spends the whole timeout again on every attempt. +_RETRYABLE_TRANSPORT_ERRORS = (httpx.ConnectError, httpx.ConnectTimeout, httpx.RemoteProtocolError, httpx.PoolTimeout) + + +def _is_retryable(exc: Exception) -> bool: + if isinstance(exc, httpx.HTTPStatusError): + return exc.response.status_code in RETRYABLE_STATUS_CODES + return isinstance(exc, _RETRYABLE_TRANSPORT_ERRORS) + + +def _request_body(ti: TakeawayInput) -> dict[str, Any]: + body: dict[str, Any] = {} + if ti.agent_id is not None: + body["agentId"] = ti.agent_id + instructions = { + key: value + for key, value in ( + ("document", ti.instructions.document), + ("slot", ti.instructions.slot), + ("formatHint", ti.instructions.format_hint), + ) + if value is not None + } + if instructions: + body["instructions"] = instructions + if ti.max_characters is not None: + body["maxCharacters"] = ti.max_characters + body["data"] = { + "visualizations": [_visualization_body(v) for v in ti.visualizations], + "filterContext": ti.filter_context, + } + return body + + +class TakeawayClient: + """Single-shot client for the takeaway endpoint.""" + + def __init__(self, host: str, token: str, workspace_id: str, *, timeout: float = 120.0): + self._url = f"{host.rstrip('/')}{_PATH.format(workspace_id=workspace_id)}" + self._auth = {"Authorization": f"Bearer {token}"} + self._client = httpx.Client(timeout=timeout) + + def ask(self, item: DatasetItem) -> ChatResult: + """Request a takeaway for one dataset item and adapt it to a ChatResult. + + Raises: + ValueError: The item carries no `takeaway_input`. + httpx.HTTPStatusError: The endpoint answered non-2xx; the message carries the + status code and the response body. Transient statuses and connection + failures are retried first, with the chat client's backoff and budget. + """ + ti = item.takeaway_input + if ti is None: + raise ValueError(f"document_takeaway item '{item.id}' is missing required 'takeaway_input'.") + + body = _request_body(ti) + + def _post() -> httpx.Response: + resp = self._client.post(self._url, json=body, headers={**self._auth, "Content-Type": "application/json"}) + if not resp.is_success: + raise httpx.HTTPStatusError( + f"Takeaway request failed with {resp.status_code} for {self._url}: {resp.text}", + request=resp.request, + response=resp, + ) + return resp + + data = retry_transient(_post, is_retryable=_is_retryable).json() + return ChatResult(text_response=data.get("takeaway") or "", raw_response=data) + + def close(self) -> None: + self._client.close() diff --git a/packages/gooddata-eval/tests/fixtures/takeaway_calibration.json b/packages/gooddata-eval/tests/fixtures/takeaway_calibration.json new file mode 100644 index 000000000..6ef3502de --- /dev/null +++ b/packages/gooddata-eval/tests/fixtures/takeaway_calibration.json @@ -0,0 +1,68 @@ +{ + "question": "Takeaway for one dashboard page", + "texts": [ + { + "label": "new markets (reference good example)", + "want": "pass", + "text": "**Growth now comes from new markets.** Sales rose again this period, led by the Nordics, while core markets held flat." + }, + { + "label": "returning customers", + "want": "pass", + "text": "**Returning customers are the gap.** Almost all active customers are new, and the biggest spenders carry a large share of sales." + }, + { + "label": "bigger baskets", + "want": "pass", + "text": "**Fewer orders, bigger baskets.** Order count dipped while the average order grew, keeping total sales steady." + }, + { + "label": "category concentration", + "want": "pass", + "text": "**Electronics carries the product mix.** It leads every other category in sales, and the gap to the rest keeps widening." + }, + { + "label": "rough size in words (prompt phrasing)", + "want": "pass", + "text": "**Audio and video now carry the store.** They bring in about a third of sales, and their sales roughly doubled compared with last period." + }, + { + "label": "executive summary", + "want": "fail", + "criteria": ["executive_summary", "exact_figure", "date_or_period"], + "text": "**Summary:** Net Sales were $4.2M in Q3 2026, up 12%. Key takeaways: Electronics leads with 38% share; returns fell to 2.1%. Things to watch: inventory in Furniture is rising, and discounts may erode margin next quarter." + }, + { + "label": "chart walk-through", + "want": "fail", + "criteria": ["chart_walkthrough", "exact_figure"], + "text": "**Sales overview.** Total Sales Trend shows sales in the low $100Ks, Net Sales by Product Category shows Audio & Video leading, and Order Breakdown shows most orders are Processed." + }, + { + "label": "hedged figures (real output)", + "want": "fail", + "criteria": ["exact_figure"], + "text": "**Solid top-line scale** Total Sales are about $270M this period, supported by a customer base of roughly 1.2K Total Customers." + }, + { + "label": "percentage with a tilde (real output)", + "want": "fail", + "criteria": [ + "exact_figure" + ], + "text": "**Sales mix:** Audio & Video Accessories drive ~40% of Net Sales, and most orders are Processed." + }, + { + "label": "page description opener", + "want": "fail", + "criteria": ["background_opener"], + "text": "This page shows sales, orders and customers for the online store. Sales rose while orders held flat." + }, + { + "label": "external cause", + "want": "fail", + "criteria": ["unshown_cause"], + "text": "**Sales jumped in the West.** The rise came because the summer campaign drew in new buyers." + } + ] +} diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index 8adff1fae..2a53f5cdc 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -2,6 +2,7 @@ import io from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone +from unittest.mock import MagicMock import httpx import orjson @@ -12,6 +13,7 @@ from gooddata_eval.core.connection import ( ConnectionError_, # noqa: F401 - used in test_cli_operational_error_exits_nonzero ) +from gooddata_eval.core.dataset.langfuse_source import _item_from_raw from gooddata_eval.core.models import DatasetItem from gooddata_eval.core.runner import EvalReport, ItemReport from gooddata_eval.core.timing import TIMERS_ENV_VAR, timers_enabled @@ -1146,3 +1148,52 @@ def test_no_warning_when_there_are_no_agentic_items(monkeypatch, tmp_path, capsy cli_main._warn_if_local_dataset_cannot_link(_local_dataset_config(tmp_path), []) assert capsys.readouterr().err == "" + + +def _routing_backend() -> tuple[cli_main._RoutingBackend, MagicMock, MagicMock, MagicMock]: + chat, summary, takeaway = MagicMock(), MagicMock(), MagicMock() + backend = cli_main._RoutingBackend(chat, {"dashboard_summary": summary, "document_takeaway": takeaway}) + return backend, chat, summary, takeaway + + +def test_cli_routes_takeaway_items_to_takeaway_client() -> None: + backend, chat, summary, takeaway = _routing_backend() + raw = { + "id": "lf-tk-1", + "datasetName": "agent_document_takeaway", + "input": "P01 default format, Sales page", + "metadata": {"takeaway_input": {"visualizations": ["v1", "v2"], "max_characters": 300}}, + "expectedOutput": {"must_include": ["States one finding"]}, + } + item = _item_from_raw(raw, dataset_name="agent_document_takeaway", test_kind="visualization") + + result = backend.ask(item) + + takeaway.ask.assert_called_once_with(item) + assert result is takeaway.ask.return_value + chat.ask.assert_not_called() + summary.ask.assert_not_called() + + +def test_cli_routes_other_kinds_by_test_kind() -> None: + backend, chat, summary, takeaway = _routing_backend() + summary_item = DatasetItem( + id="s", dataset_name="d", test_kind="dashboard_summary", question="q", expected_output={} + ) + chat_item = DatasetItem(id="c", dataset_name="d", test_kind="general_question", question="q", expected_output="x") + + backend.ask(summary_item) + backend.ask(chat_item) + + summary.ask.assert_called_once_with(summary_item) + chat.ask.assert_called_once_with(chat_item) + takeaway.ask.assert_not_called() + + +def test_routing_backend_closes_every_client() -> None: + backend, chat, summary, takeaway = _routing_backend() + + backend.close() + + for client in (chat, summary, takeaway): + client.close.assert_called_once_with() diff --git a/packages/gooddata-eval/tests/test_langfuse_source.py b/packages/gooddata-eval/tests/test_langfuse_source.py index 92448b01e..028d7d391 100644 --- a/packages/gooddata-eval/tests/test_langfuse_source.py +++ b/packages/gooddata-eval/tests/test_langfuse_source.py @@ -73,6 +73,102 @@ def test_item_from_raw_summary_input_absent_is_none(): assert item.summary_input is None +def test_langfuse_item_takeaway_input_from_input_object() -> None: + raw = { + "id": "lf-tk-1", + "datasetName": "ds", + "input": {"question": "Takeaway for the page", "takeaway_input": {"visualizations": ["v1"]}}, + "metadata": {"takeaway_input": {"visualizations": ["from-metadata"]}}, + "expectedOutput": {"test_kind": "document_takeaway"}, + } + item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization") + assert item.takeaway_input is not None + assert item.takeaway_input.visualizations == ["v1"] + + +def test_langfuse_item_takeaway_input_from_metadata() -> None: + raw = { + "id": "lf-tk-2", + "datasetName": "ds", + "input": "Takeaway for the page", + "metadata": {"takeaway_input": {"visualizations": ["v2"], "max_characters": 1000}}, + "expectedOutput": {"must_include": ["revenue"]}, + } + item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization") + assert item.test_kind == "document_takeaway" + assert item.takeaway_input.visualizations == ["v2"] + assert item.takeaway_input.max_characters == 1000 + + +def test_langfuse_item_takeaway_input_from_expected_output() -> None: + raw = { + "id": "lf-tk-5", + "datasetName": "ds", + "input": "Takeaway for the page", + "expectedOutput": {"takeaway_input": {"visualizations": ["v3"]}}, + } + item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization") + assert item.test_kind == "document_takeaway" + assert item.takeaway_input.visualizations == ["v3"] + + +def test_langfuse_item_structural_kind_beats_takeaway_input() -> None: + raw = { + "id": "lf-tk-6", + "datasetName": "ds", + "input": {"question": "Show revenue", "takeaway_input": {"visualizations": ["v1"]}}, + "expectedOutput": {"visualization": {"id": "x", "type": "", "query": {"fields": {}}}}, + } + item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization") + assert item.test_kind == "vis_agentic" + + +def test_langfuse_item_invalid_takeaway_input_names_the_item() -> None: + raw = { + "id": "lf-tk-bad", + "datasetName": "ds", + "input": {"question": "Takeaway", "takeaway_input": {"visualizations": ["v1"], "max_chars": 400}}, + "expectedOutput": {}, + } + with pytest.raises(ValueError, match=r"(?s)'lf-tk-bad'.*max_chars"): + _item_from_raw(raw, dataset_name="ds", test_kind="visualization") + + +@pytest.mark.parametrize("takeaway_input", ["v1", ["v1"]]) +def test_langfuse_item_non_object_takeaway_input_names_the_item(takeaway_input: object) -> None: + raw = { + "id": "lf-tk-shape", + "datasetName": "ds", + "input": "Takeaway", + "metadata": {"takeaway_input": takeaway_input}, + "expectedOutput": {}, + } + with pytest.raises(ValueError, match="'lf-tk-shape'.*takeaway_input"): + _item_from_raw(raw, dataset_name="ds", test_kind="visualization") + + +def test_langfuse_item_infers_document_takeaway_kind() -> None: + raw = { + "id": "lf-tk-3", + "datasetName": "ds", + "input": {"question": "Takeaway for the page", "takeaway_input": {"visualizations": ["v1"]}}, + "expectedOutput": {"no_data": False, "must_include": ["revenue"]}, + } + item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization") + assert item.test_kind == "document_takeaway" + + +def test_langfuse_item_explicit_kind_beats_takeaway_input() -> None: + raw = { + "id": "lf-tk-4", + "datasetName": "ds", + "input": {"question": "Takeaway for the page", "takeaway_input": {"visualizations": ["v1"]}}, + "expectedOutput": {"test_kind": "agentic_general_question"}, + } + item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization") + assert item.test_kind == "agentic_general_question" + + def test_load_langfuse_dataset_calls_rest_api(monkeypatch): monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test") monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test") diff --git a/packages/gooddata-eval/tests/test_models.py b/packages/gooddata-eval/tests/test_models.py index 0c09a23ef..a34cc2a58 100644 --- a/packages/gooddata-eval/tests/test_models.py +++ b/packages/gooddata-eval/tests/test_models.py @@ -1,9 +1,12 @@ # (C) 2026 GoodData Corporation +import pytest from gooddata_eval.core.models import ( ChatResult, CreatedVisualization, DatasetItem, ReasoningStepEvent, + TakeawayInput, + TakeawayVisualization, ToolCallEvent, build_tool_calls, timeline_detail, @@ -171,3 +174,50 @@ def test_a_visualization_without_an_id_still_parses(): """The agent sometimes omits `id`; nothing scores on it, so it must not error the item.""" viz = CreatedVisualization.model_validate({"type": "bar_chart", "query": {"fields": {"m": "metric/x"}}}) assert viz.id is None + + +def test_takeaway_input_accepts_id_and_placement_visualizations() -> None: + takeaway_input = TakeawayInput.model_validate( + { + "visualizations": ["v1", {"visualization_id": "v2", "filters": [{"positiveAttributeFilter": {}}]}], + "instructions": {"slot": "page", "format_hint": "one sentence"}, + "max_characters": 400, + } + ) + assert takeaway_input.visualizations == [ + "v1", + TakeawayVisualization(visualization_id="v2", filters=[{"positiveAttributeFilter": {}}]), + ] + assert takeaway_input.filter_context == [] + assert takeaway_input.instructions.format_hint == "one sentence" + assert takeaway_input.max_characters == 400 + assert takeaway_input.agent_id is None + + +def test_takeaway_input_rejects_unknown_key() -> None: + with pytest.raises(ValueError, match="max_chars"): + TakeawayInput.model_validate({"visualizations": ["v1"], "max_chars": 400}) + + +@pytest.mark.parametrize( + "raw", + [ + {"visualizations": [{"visualization_id": "v1", "filter": []}]}, + {"visualizations": ["v1"], "instructions": {"formatHint": "one sentence"}}, + ], +) +def test_takeaway_input_rejects_unknown_nested_key(raw: dict) -> None: + with pytest.raises(ValueError, match="Extra inputs are not permitted"): + TakeawayInput.model_validate(raw) + + +def test_takeaway_input_requires_a_visualization() -> None: + with pytest.raises(ValueError, match="visualizations"): + TakeawayInput.model_validate({"visualizations": []}) + + +def test_dataset_item_takeaway_input_defaults_to_none() -> None: + item = DatasetItem.model_validate( + {"id": "q1", "dataset_name": "d1", "test_kind": "dashboard_summary", "question": "q", "expected_output": {}} + ) + assert item.takeaway_input is None diff --git a/packages/gooddata-eval/tests/test_runner.py b/packages/gooddata-eval/tests/test_runner.py index eb7a2bad2..6094621b8 100644 --- a/packages/gooddata-eval/tests/test_runner.py +++ b/packages/gooddata-eval/tests/test_runner.py @@ -134,6 +134,7 @@ def test_run_items_routes_all_supported_kinds(): "guardrail", "dashboard_summary", "knowledge_question", + "document_takeaway", } assert expected_kinds == supported_test_kinds() diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index 79f9cb818..53982fbf3 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -6,6 +6,7 @@ import httpx import pytest +from gooddata_eval.core import _retry as retry_mod from gooddata_eval.core.chat import sse_client as sse_mod from gooddata_eval.core.chat.sse_client import ( ChatClient, @@ -1350,3 +1351,8 @@ def test_a_turn_that_fails_after_response_started_still_counts_as_joined(monkeyp client.send_message("conv-failed-turn", "q") assert joined_turns("conv-failed-turn") == 1 + + +def test_retry_names_external_callers_import_stay_on_sse_client() -> None: + assert sse_mod._RETRYABLE_STATUS_CODES is retry_mod.RETRYABLE_STATUS_CODES + assert sse_mod._retry_transient is retry_mod.retry_transient diff --git a/packages/gooddata-eval/tests/test_takeaway_client.py b/packages/gooddata-eval/tests/test_takeaway_client.py new file mode 100644 index 000000000..5fe7b084d --- /dev/null +++ b/packages/gooddata-eval/tests/test_takeaway_client.py @@ -0,0 +1,257 @@ +# (C) 2026 GoodData Corporation +import json +from collections.abc import Callable + +import httpx +import pytest +from gooddata_eval.core import _retry as retry_mod +from gooddata_eval.core.models import DatasetItem, TakeawayInput +from gooddata_eval.core.takeaway.http_client import TakeawayClient + +_TAKEAWAY_RESPONSE = { + "takeaway": "**Revenue grew 12%.** Driven by the West region.", + "noData": False, + "generatedAt": "2026-10-22T10:00:00Z", + "agentId": "agent-1", + "contextOmitted": [], +} + + +def _item(takeaway_input: dict | None) -> DatasetItem: + return DatasetItem( + id="t1", + dataset_name="d", + test_kind="document_takeaway", + question="Takeaway for the page", + expected_output={}, + takeaway_input=TakeawayInput.model_validate(takeaway_input) if takeaway_input is not None else None, + ) + + +def _client(handler: Callable[[httpx.Request], httpx.Response]) -> TakeawayClient: + client = TakeawayClient(host="https://h/", token="tok", workspace_id="ws") + client._client = httpx.Client(transport=httpx.MockTransport(handler)) + return client + + +def _capturing(captured: dict, response: dict | None = None) -> Callable[[httpx.Request], httpx.Response]: + def handler(request: httpx.Request) -> httpx.Response: + captured["url"] = str(request.url) + captured["auth"] = request.headers.get("authorization") + captured["body"] = json.loads(request.content) + return httpx.Response(200, json=response or _TAKEAWAY_RESPONSE) + + return handler + + +def test_body_is_camel_case_and_omits_unset() -> None: + captured: dict = {} + client = _client(_capturing(captured)) + + result = client.ask( + _item( + { + "visualizations": ["v1"], + "filter_context": [{"dateFilter": {}}], + "instructions": {"slot": "page", "format_hint": "one sentence"}, + "max_characters": 1000, + "agent_id": "agent-1", + } + ) + ) + + assert captured["url"] == "https://h/api/v1/ai/workspaces/ws/takeaway" + assert captured["auth"] == "Bearer tok" + assert captured["body"] == { + "agentId": "agent-1", + "instructions": {"slot": "page", "formatHint": "one sentence"}, + "maxCharacters": 1000, + "data": {"visualizations": ["v1"], "filterContext": [{"dateFilter": {}}]}, + } + assert result.text_response == "**Revenue grew 12%.** Driven by the West region." + + +def test_placement_visualization_serialized_with_filters() -> None: + captured: dict = {} + client = _client(_capturing(captured)) + + client.ask( + _item( + { + "visualizations": [ + "v1", + {"visualization_id": "v2", "filters": [{"positiveAttributeFilter": {}}]}, + {"visualization_id": "v3"}, + ] + } + ) + ) + + assert captured["body"] == { + "data": { + "visualizations": [ + "v1", + {"visualizationId": "v2", "filters": [{"positiveAttributeFilter": {}}]}, + {"visualizationId": "v3", "filters": []}, + ], + "filterContext": [], + }, + } + + +def test_instructions_omitted_when_empty() -> None: + captured: dict = {} + client = _client(_capturing(captured)) + + client.ask(_item({"visualizations": ["v1"], "instructions": {}})) + + assert "instructions" not in captured["body"] + assert captured["body"]["data"]["filterContext"] == [] + + +def test_no_data_response_kept_as_no_data_not_empty_text() -> None: + no_data = {**_TAKEAWAY_RESPONSE, "takeaway": None, "noData": True, "contextOmitted": ["v1"]} + client = _client(_capturing({}, no_data)) + + result = client.ask(_item({"visualizations": ["v1"]})) + + assert result.text_response == "" + assert result.raw_response is not None + assert result.raw_response["noData"] is True + assert result.raw_response["takeaway"] is None + assert result.raw_response["agentId"] == "agent-1" + assert result.raw_response["contextOmitted"] == ["v1"] + + +def test_http_error_carries_status_and_body(sleeps: list[float]) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(502, json={"detail": "Visualizations ['v1'] failed to execute."}) + + client = _client(handler) + + with pytest.raises(httpx.HTTPStatusError) as exc_info: + client.ask(_item({"visualizations": ["v1"]})) + + assert exc_info.value.response.status_code == 502 + assert "502" in str(exc_info.value) + assert "failed to execute" in str(exc_info.value) + + +def test_redirect_is_an_http_error() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(302, headers={"Location": "https://h/login"}, text="Found") + + client = _client(handler) + + with pytest.raises(httpx.HTTPStatusError, match="302"): + client.ask(_item({"visualizations": ["v1"]})) + + +def test_item_without_takeaway_input_names_the_item() -> None: + client = TakeawayClient(host="https://h", token="tok", workspace_id="ws") + + with pytest.raises(ValueError, match="'t1'.*takeaway_input"): + client.ask(_item(None)) + + +@pytest.fixture +def sleeps(monkeypatch: pytest.MonkeyPatch) -> list[float]: + for name in ( + "GOODDATA_EVAL_CHAT_MAX_RETRIES", + "GOODDATA_EVAL_CHAT_INITIAL_BACKOFF_S", + "GOODDATA_EVAL_CHAT_BACKOFF_FACTOR", + "GOODDATA_EVAL_CHAT_MAX_BACKOFF_S", + ): + monkeypatch.delenv(name, raising=False) + recorded: list[float] = [] + monkeypatch.setattr(retry_mod.time, "sleep", recorded.append) + return recorded + + +def _scripted(*responses: httpx.Response | Exception) -> tuple[Callable[[httpx.Request], httpx.Response], list[int]]: + calls: list[int] = [] + it = iter(responses) + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(1) + nxt = next(it) + if isinstance(nxt, Exception): + raise nxt + return nxt + + return handler, calls + + +@pytest.mark.parametrize("status", [429, 502, 503, 504]) +def test_transient_status_is_retried(status: int, sleeps: list[float]) -> None: + handler, calls = _scripted(httpx.Response(status, text="busy"), httpx.Response(200, json=_TAKEAWAY_RESPONSE)) + + result = _client(handler).ask(_item({"visualizations": ["v1"]})) + + assert result.text_response == _TAKEAWAY_RESPONSE["takeaway"] + assert len(calls) == 2 + assert sleeps == [5.0] + + +@pytest.mark.parametrize( + "error", + [ + httpx.ConnectError("refused"), + httpx.ConnectTimeout("connect timed out"), + httpx.RemoteProtocolError("peer closed connection"), + httpx.PoolTimeout("pool exhausted"), + ], +) +def test_transient_transport_error_is_retried(error: httpx.TransportError, sleeps: list[float]) -> None: + handler, calls = _scripted(error, httpx.Response(200, json=_TAKEAWAY_RESPONSE)) + + _client(handler).ask(_item({"visualizations": ["v1"]})) + + assert len(calls) == 2 + + +@pytest.mark.parametrize( + "error", + [ + httpx.ReadTimeout("read timed out"), + httpx.WriteTimeout("write timed out"), + httpx.UnsupportedProtocol("no scheme"), + ], +) +def test_slow_or_misconfigured_request_is_not_retried(error: httpx.TransportError, sleeps: list[float]) -> None: + handler, calls = _scripted(error) + + with pytest.raises(type(error)): + _client(handler).ask(_item({"visualizations": ["v1"]})) + + assert len(calls) == 1 + assert sleeps == [] + + +def test_transient_status_raises_with_body_once_retries_run_out(sleeps: list[float]) -> None: + handler, calls = _scripted(*[httpx.Response(502, text="Visualizations failed to execute.")] * 6) + + with pytest.raises(httpx.HTTPStatusError, match="502.*failed to execute") as exc_info: + _client(handler).ask(_item({"visualizations": ["v1"]})) + + assert exc_info.value.response.status_code == 502 + assert len(calls) == 6 + + +@pytest.mark.parametrize("status", [400, 404, 422]) +def test_deterministic_client_error_is_not_retried(status: int, sleeps: list[float]) -> None: + handler, calls = _scripted(httpx.Response(status, text='{"reason": "NO_AVAILABLE_AGENT"}')) + + with pytest.raises(httpx.HTTPStatusError, match="NO_AVAILABLE_AGENT"): + _client(handler).ask(_item({"visualizations": ["v1"]})) + + assert len(calls) == 1 + assert sleeps == [] + + +def test_default_timeout_is_two_minutes() -> None: + assert TakeawayClient(host="https://h", token="tok", workspace_id="ws")._client.timeout.read == 120.0 + + +def test_timeout_is_overridable() -> None: + assert TakeawayClient(host="https://h", token="tok", workspace_id="ws", timeout=30.0)._client.timeout.read == 30.0 diff --git a/packages/gooddata-eval/tests/test_takeaway_evaluate.py b/packages/gooddata-eval/tests/test_takeaway_evaluate.py new file mode 100644 index 000000000..112d98df4 --- /dev/null +++ b/packages/gooddata-eval/tests/test_takeaway_evaluate.py @@ -0,0 +1,349 @@ +# (C) 2026 GoodData Corporation +import inspect +from typing import Any +from unittest.mock import MagicMock, patch + +import httpx +import pytest +from gooddata_eval.core.agentic import _langfuse as agentic_langfuse +from gooddata_eval.core.agentic._trace_linker import run_trace_link_inline +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError +from gooddata_eval.core.evaluators.takeaway import DocumentTakeawayEvaluator +from gooddata_eval.core.models import ChatResult +from gooddata_eval.core.takeaway.evaluate import TakeawayAssertionError, evaluate_document_takeaway + +from tests._fake_langfuse import FakeLangfuse + +_GOOD = "**Growth now comes from new markets.** Sales rose again this period, led by the Nordics, while core markets held flat." +_LONG = "**Sales grew.** " + " ".join(["word"] * 60) + "." +_OTLP = "/api/public/otel/v1/traces" +_SCORES = "/api/public/scores" + + +def _posted_scores(fake_langfuse: FakeLangfuse) -> list[dict]: + bodies: list[dict] = [] + for call in fake_langfuse.calls("POST", _SCORES): + body = call["json"] + bodies.extend(body if isinstance(body, list) else [body]) + return bodies + + +def _dataset_item(**overrides: Any) -> dict: + return { + "id": "item-1", + "question": "P01 default format, Sales page", + "expected_output": {}, + "takeaway_input": {"visualizations": ["v1"], "max_characters": 300}, + **overrides, + } + + +def _chat(text: str) -> ChatResult: + return ChatResult(text_response=text, raw_response={"takeaway": text, "noData": False, "contextOmitted": []}) + + +def _client(*texts: str) -> MagicMock: + client = MagicMock() + client.ask.side_effect = [_chat(t) for t in texts] + return client + + +def _evaluator(*, judge_error: bool = False) -> DocumentTakeawayEvaluator: + with patch("openai.OpenAI"), patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}): + ev = DocumentTakeawayEvaluator() + for judge, verdict in ((ev._positive_judge, True), (ev._violation_judge, False)): + judge.score = MagicMock( + side_effect=JudgeResponseError("empty body twice") if judge_error else None, return_value=(verdict, "ok") + ) + return ev + + +def _evaluate(client: MagicMock | None, **kwargs: Any) -> Any: + params: dict[str, Any] = { + "host": "https://h", + "token": "tok", + "workspace_id": "ws", + "dataset_item": _dataset_item(), + "dataset_name": "agent_document_takeaway", + "run_timestamp": "2026-10-22_00-00-00", + "client": client, + "evaluator": _evaluator(), + "langfuse": MagicMock(), + **kwargs, + } + return evaluate_document_takeaway(**params) + + +def test_evaluate_builds_and_closes_its_own_client_when_none_is_given() -> None: + client = _client(_GOOD) + with patch("gooddata_eval.core.takeaway.evaluate.TakeawayClient", return_value=client) as client_cls: + outcome = _evaluate(None) + + client_cls.assert_called_once_with(host="https://h", token="tok", workspace_id="ws") + client.close.assert_called_once_with() + assert outcome.runs_passed == 1 + + +def test_evaluate_leaves_an_injected_client_open() -> None: + client = _client(_GOOD) + _evaluate(client) + + client.close.assert_not_called() + + +def test_evaluate_passes_and_returns_the_outcome() -> None: + outcome = _evaluate(_client(_GOOD, _GOOD), k=2) + + assert outcome.runs_passed == 2 + assert outcome.runs_effective == 2 + assert outcome.detail["word_count"] == 20 + assert outcome.failed_runs == [] + assert outcome.timings.agent_s >= 0.0 + + +def test_evaluate_raises_on_fail_with_detail() -> None: + with pytest.raises(TakeawayAssertionError) as exc_info: + _evaluate(_client(_LONG)) + + exc = exc_info.value + assert "word_budget" in str(exc) + assert "Gate pass@1 failed: 0/1 runs passed" in str(exc) + assert exc.detail["word_budget"] is False + assert exc.detail["word_count"] == 62 + assert exc.runs_passed == 0 + assert exc.runs_effective == 1 + assert len(exc.failed_runs) == 1 + + +def test_evaluate_power_gate_fails_on_one_bad_run() -> None: + with pytest.raises(TakeawayAssertionError, match="pass\\^2 failed: 1/2"): + _evaluate(_client(_GOOD, _LONG), k=2, gate="power") + + outcome = _evaluate(_client(_GOOD, _LONG), k=2, gate="any") + assert outcome.runs_passed == 1 + assert len(outcome.failed_runs) == 1 + + +def test_every_run_ungraded_is_a_judge_error_not_a_failure() -> None: + with pytest.raises(JudgeResponseError, match="no readable verdict") as exc_info: + _evaluate(_client(_GOOD, _GOOD), k=2, evaluator=_evaluator(judge_error=True)) + + exc = exc_info.value + assert exc.runs_effective == 2 + assert exc.runs_passed == 0 + assert exc.detail["actual_output"] == _GOOD + assert exc.detail["word_count"] == 20 + assert exc.detail["unscored_runs"] == 2 + assert [run["error"] for run in exc.failed_runs] == [exc.detail["judge_errors"][0]] * 2 + + +def test_item_without_takeaway_input_is_rejected_before_any_request() -> None: + client = _client(_GOOD) + item = _dataset_item() + del item["takeaway_input"] + + with pytest.raises(ValueError, match="'item-1'.*takeaway_input"): + _evaluate(client, dataset_item=item) + + client.ask.assert_not_called() + + +def test_http_error_surfaces_with_status_and_body() -> None: + request = httpx.Request("POST", "https://h/api/v1/ai/workspaces/ws/takeaway") + response = httpx.Response(400, request=request, text='{"reason": "NO_AVAILABLE_AGENT"}') + client = MagicMock() + client.ask.side_effect = httpx.HTTPStatusError( + f"Takeaway request failed with 400: {response.text}", request=request, response=response + ) + submit = MagicMock() + + with pytest.raises(httpx.HTTPStatusError, match="NO_AVAILABLE_AGENT"): + _evaluate(client, submit_trace_link=submit) + + submit.assert_not_called() + + +def test_signature_accepts_what_the_tavern_shim_passes() -> None: + params = inspect.signature(evaluate_document_takeaway).parameters + + for name in ( + "host", + "token", + "workspace_id", + "dataset_item", + "dataset_name", + "run_timestamp", + "model_version_override", + "reasoning_effort", + "k", + "gate", + "run_metadata_extra", + ): + assert name in params, name + + +def test_langfuse_item_has_testing_framework_metadata(fake_langfuse: FakeLangfuse) -> None: + with patch.object(agentic_langfuse, "get_model_version", return_value="gpt-5.2"): + _evaluate( + _client(_GOOD), + langfuse=None, + submit_trace_link=run_trace_link_inline, + run_metadata_extra={"testing_framework": "tavern-e2e", "github_run_id": "123"}, + ) + + spans = [ + span + for call in fake_langfuse.calls("POST", _OTLP) + for resource in call["json"]["resourceSpans"] + for scope in resource["scopeSpans"] + for span in scope["spans"] + ] + assert len(spans) == 1 + attrs = {a["key"]: next(iter(a["value"].values())) for a in spans[0]["attributes"]} + assert attrs["langfuse.experiment.metadata.testing_framework"] == "tavern-e2e" + assert attrs["langfuse.experiment.metadata.github_run_id"] == "123" + assert attrs["langfuse.experiment.metadata.model_version"] == "gpt-5.2" + + scores = {score["name"]: score for score in _posted_scores(fake_langfuse)} + for name in ("gate_passed", "pass_at_k", "pass_power_k", "word_budget", "no_meta", "quality_score"): + assert name in scores, name + assert scores["word_count"]["value"] == 20 + assert scores["word_count"]["dataType"] == "NUMERIC" + assert scores["headline_word_count"]["value"] == 6 + assert scores["default_exclude_exact_figure"]["value"] == 1.0 + + +def _http_error(status: int, body: str) -> httpx.HTTPStatusError: + request = httpx.Request("POST", "https://h/api/v1/ai/workspaces/ws/takeaway") + response = httpx.Response(status, request=request, text=body) + return httpx.HTTPStatusError(f"Takeaway request failed with {status}: {body}", request=request, response=response) + + +def test_a_transient_error_on_one_run_keeps_the_graded_runs(fake_langfuse: FakeLangfuse) -> None: + client = MagicMock() + client.ask.side_effect = [_chat(_GOOD), _chat(_GOOD), _http_error(502, "Visualizations failed to execute.")] + + with patch.object(agentic_langfuse, "get_model_version", return_value="gpt-5.2"): + outcome = _evaluate(client, k=3, gate="any", langfuse=None, submit_trace_link=run_trace_link_inline) + + assert outcome.runs_passed == 2 + assert outcome.runs_effective == 3 + assert outcome.detail["unscored_runs"] == 1 + assert outcome.detail["request_errors"] == [outcome.failed_runs[0]["error"]] + assert "judge_errors" not in outcome.detail + assert len(outcome.failed_runs) == 1 + assert "502" in outcome.failed_runs[0]["error"] + assert "failed to execute" in outcome.failed_runs[0]["error"] + gate_scores = [s for s in _posted_scores(fake_langfuse) if s["name"] == "gate_passed"] + assert len(gate_scores) == 2 + assert all(score["value"] == 1 for score in gate_scores) + + +def test_a_transient_error_on_every_run_raises_it() -> None: + client = MagicMock() + client.ask.side_effect = [_http_error(502, "Visualizations failed to execute.")] * 2 + submit = MagicMock() + + with pytest.raises(httpx.HTTPStatusError, match="502.*failed to execute"): + _evaluate(client, k=2, submit_trace_link=submit) + + submit.assert_not_called() + + +def test_a_deterministic_client_error_raises_on_the_first_run() -> None: + client = MagicMock() + client.ask.side_effect = [_http_error(400, '{"reason": "NO_AVAILABLE_AGENT"}'), _chat(_GOOD), _chat(_GOOD)] + + with pytest.raises(httpx.HTTPStatusError, match="NO_AVAILABLE_AGENT"): + _evaluate(client, k=3) + + assert client.ask.call_count == 1 + + +def test_no_data_run_publishes_no_word_counts(fake_langfuse: FakeLangfuse) -> None: + client = MagicMock() + client.ask.return_value = ChatResult(text_response="", raw_response={"takeaway": None, "noData": True}) + + with patch.object(agentic_langfuse, "get_model_version", return_value="gpt-5.2"): + _evaluate( + client, + dataset_item=_dataset_item(expected_output={"no_data": True}), + langfuse=None, + submit_trace_link=run_trace_link_inline, + ) + + names = {score["name"] for score in _posted_scores(fake_langfuse)} + assert "no_data" in names + assert "word_count" not in names + assert "headline_word_count" not in names + + +def test_an_invalid_fixture_fails_before_any_request() -> None: + client = _client(_GOOD) + submit = MagicMock() + + with pytest.raises(ValueError, match="max_sentence"): + _evaluate( + client, + dataset_item=_dataset_item(expected_output={"format": {"max_sentence": 1}}), + submit_trace_link=submit, + ) + + client.ask.assert_not_called() + submit.assert_not_called() + + +def test_each_run_writes_its_own_document_takeaway_pass(fake_langfuse: FakeLangfuse) -> None: + with patch.object(agentic_langfuse, "get_model_version", return_value="gpt-5.2"): + _evaluate(_client(_GOOD, _LONG), k=2, gate="any", langfuse=None, submit_trace_link=run_trace_link_inline) + + scores = [s for s in _posted_scores(fake_langfuse) if s["name"] == "document_takeaway_pass"] + assert sorted(score["value"] for score in scores) == [0, 1] + assert all(score["dataType"] == "BOOLEAN" for score in scores) + + +def test_a_read_timeout_is_an_errored_run_not_an_item_failure() -> None: + client = MagicMock() + client.ask.side_effect = [_chat(_GOOD), httpx.ReadTimeout("read timed out")] + + outcome = _evaluate(client, k=2, gate="any") + + assert outcome.runs_passed == 1 + assert "ReadTimeout" in outcome.detail["request_errors"][0] + + +def test_an_unsupported_protocol_raises_on_the_first_run() -> None: + client = MagicMock() + client.ask.side_effect = [httpx.UnsupportedProtocol("no scheme"), _chat(_GOOD)] + + with pytest.raises(httpx.UnsupportedProtocol): + _evaluate(client, k=2) + + assert client.ask.call_count == 1 + + +def test_power_gate_failure_from_an_errored_run_names_the_request_error() -> None: + client = MagicMock() + client.ask.side_effect = [_chat(_GOOD), _http_error(502, "Visualizations failed to execute.")] + + with pytest.raises(TakeawayAssertionError) as exc_info: + _evaluate(client, k=2, gate="power") + + message = str(exc_info.value) + assert "Failed checks: none" not in message + assert "Request errors:" in message + assert "502" in message + + +def test_an_ungraded_item_with_request_errors_says_so() -> None: + client = MagicMock() + client.ask.side_effect = [_chat(_GOOD), _http_error(502, "Visualizations failed to execute.")] + + with pytest.raises(JudgeResponseError) as exc_info: + _evaluate(client, k=2, evaluator=_evaluator(judge_error=True)) + + message = str(exc_info.value) + assert "no run of 2 could be graded" in message + assert "judge returned no readable verdict for 1 run(s)" in message + assert "request failed for 1 run(s)" in message + assert "502" in message diff --git a/packages/gooddata-eval/tests/test_takeaway_evaluator.py b/packages/gooddata-eval/tests/test_takeaway_evaluator.py new file mode 100644 index 000000000..26a93d780 --- /dev/null +++ b/packages/gooddata-eval/tests/test_takeaway_evaluator.py @@ -0,0 +1,425 @@ +# (C) 2026 GoodData Corporation +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.evaluators import get_evaluator, supported_test_kinds +from gooddata_eval.core.evaluators.takeaway import ( + DEFAULT_MUST_INCLUDE, + DEFAULT_MUST_NOT_INCLUDE, + DocumentTakeawayEvaluator, + _sentences, +) +from gooddata_eval.core.models import ChatResult, DatasetItem, TakeawayInput + +_GOOD = "**Growth now comes from new markets.** Sales rose again this period, led by the Nordics, while core markets held flat." + + +def _make_evaluator(*, judge_passes: bool = True) -> DocumentTakeawayEvaluator: + """An evaluator whose judges grade every criterion as satisfied (or every one as not).""" + with patch("openai.OpenAI"), patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}): + ev = DocumentTakeawayEvaluator() + ev._positive_judge.score = MagicMock(return_value=(judge_passes, "positive")) + ev._violation_judge.score = MagicMock(return_value=(not judge_passes, "violation")) + return ev + + +def _item(expected_output: Any = None, *, max_characters: int | None = None) -> DatasetItem: + return DatasetItem( + id="t1", + dataset_name="d", + test_kind="document_takeaway", + question="P01 default format, Sales page", + expected_output=expected_output if expected_output is not None else {}, + takeaway_input=TakeawayInput(visualizations=["v1"], max_characters=max_characters), + ) + + +def _chat(text: str | None = _GOOD, *, no_data: bool = False) -> ChatResult: + raw = {"takeaway": text, "noData": no_data, "agentId": "a", "contextOmitted": []} + return ChatResult(text_response=text or "", raw_response=raw) + + +def _judge_calls(ev: DocumentTakeawayEvaluator) -> int: + return ev._positive_judge.score.call_count + ev._violation_judge.score.call_count + + +def test_good_takeaway_passes_every_check() -> None: + res = _make_evaluator().evaluate(_item(), _chat()) + + assert res.passed is True + assert res.error is None + for key in ( + "word_budget", + "headline_words", + "max_sentences", + "single_paragraph", + "bold_headline", + "no_headings", + "no_lists", + "no_ids", + "no_meta", + ): + assert res.detail[key] is True, key + + +def test_word_count_scores_reported() -> None: + res = _make_evaluator().evaluate(_item(), _chat()) + + assert res.detail["word_count"] == 20 + assert res.detail["headline_word_count"] == 6 + + +def test_word_budget_ignores_large_max_characters() -> None: + text = "**Sales grew.** " + " ".join(["word"] * 53) + "." + + res = _make_evaluator().evaluate(_item(max_characters=1000), _chat(text)) + + assert res.detail["word_count"] == 55 + assert res.detail["word_budget"] is False + assert res.detail["length"] is True + assert res.passed is False + + +def test_headline_over_ten_words_fails() -> None: + text = "**Sales in every region grew faster than any other metric this time.** Margins held." + + res = _make_evaluator().evaluate(_item(), _chat(text)) + + assert res.detail["headline_word_count"] == 12 + assert res.detail["headline_words"] is False + assert res.passed is False + + +@pytest.mark.parametrize( + ("body", "count"), + [ + ("Sales rose 3% vs. last period. Nordics led.", 2), + ("U.S. sales grew. Europe held flat.", 2), + ("Prodeje rostly, např. v Praze. Brno stagnovalo.", 2), + ("Prodeje rostly. Živé kanály stagnovaly.", 2), + ("Sales rose **sharply.** Orders fell. Returns grew.", 3), + ('Sales are "flat." Orders rose.', 2), + ("Margins held at 3.5% overall.", 1), + ("Sales grew. 2 regions lead.", 2), + ("Sales grew", 1), + ("Online outgrew retail vs. Europe in 2024.", 1), + ("Sales stay below vs. 2024 levels.", 1), + ("Growth came from new markets, e.g. Nordics. Core held.", 2), + ("Tržby rostly, tj. Praha vedla. Brno stagnovalo.", 2), + ("Sales grew (cca. 5 regions). Core held.", 2), + ("... Sales grew.", 2), + ("", 0), + (" ** ** ", 0), + ], +) +def test_sentence_count(body: str, count: int) -> None: + assert _sentences(body) == count + + +def test_bold_inside_the_body_still_counts_as_a_break() -> None: + res = _make_evaluator().evaluate( + _item(), _chat("**Sales grew.** Sales rose **sharply.** Orders fell. Returns grew.") + ) + + assert res.detail["max_sentences"] is False + + +@pytest.mark.parametrize( + ("text", "ok"), + [ + ("**Sales grew.** Margins held. Costs fell.", True), + ("**Sales grew.** Margins held. Costs fell. Returns rose.", False), + ("**Total is $4.2M.** Margins held at 3.5% overall.", True), + ], +) +def test_max_sentences_counts_after_the_headline(text: str, ok: bool) -> None: + res = _make_evaluator().evaluate(_item(), _chat(text)) + + assert res.detail["max_sentences"] is ok + + +def test_max_sentences_counts_whole_text_without_headline() -> None: + res = _make_evaluator().evaluate( + _item({"format": {"bold_headline": False, "max_sentences": 1}}), _chat("Sales grew. Margins held.") + ) + + assert res.detail["max_sentences"] is False + assert "bold_headline" not in res.detail + assert "headline_words" not in res.detail + assert res.detail["headline_word_count"] == 0 + + +@pytest.mark.parametrize( + ("text", "ok"), + [ + (_GOOD, True), + ("**Sales grew.**\nMargins held.", True), + ("**Sales grew.**\n\nMargins held.", False), + ("**Sales grew.**\nMargins held.\nCosts fell.", False), + ], +) +def test_single_paragraph(text: str, ok: bool) -> None: + res = _make_evaluator().evaluate(_item(), _chat(text)) + + assert res.detail["single_paragraph"] is ok + + +@pytest.mark.parametrize(("limit", "ok"), [(len(_GOOD), True), (len(_GOOD) - 1, False)]) +def test_length_within_max_characters(limit: int, ok: bool) -> None: + res = _make_evaluator().evaluate(_item(max_characters=limit), _chat()) + + assert res.detail["length"] is ok + + +def test_length_not_checked_without_max_characters() -> None: + res = _make_evaluator().evaluate(_item(), _chat()) + + assert "length" not in res.detail + + +@pytest.mark.parametrize( + ("text", "ok"), + [ + (_GOOD, True), + (" **Sales grew.** Margins held.", True), + ("Sales grew. **Margins held.**", False), + ("Sales grew and margins held.", False), + ], +) +def test_bold_headline(text: str, ok: bool) -> None: + res = _make_evaluator().evaluate(_item(), _chat(text)) + + assert res.detail["bold_headline"] is ok + + +@pytest.mark.parametrize( + ("text", "ok"), + [ + (_GOOD, True), + ("## Sales grew\nMargins held.", False), + ("# Sales grew\nMargins held.", False), + ("#1 seller holds the lead.", True), + ("**Sales grew.**\n#2 brand closes the gap.", True), + ], +) +def test_no_headings(text: str, ok: bool) -> None: + res = _make_evaluator().evaluate(_item(), _chat(text)) + + assert res.detail["no_headings"] is ok + + +@pytest.mark.parametrize( + ("text", "ok"), + [ + (_GOOD, True), + ("**Sales grew.**\n- Margins held.", False), + ("**Sales grew.**\n• Margins held.", False), + ("**Sales grew.**\n1. Margins held.", False), + ("**Sales grew.**\n2) Margins held.", False), + ], +) +def test_no_lists(text: str, ok: bool) -> None: + res = _make_evaluator().evaluate(_item(), _chat(text)) + + assert res.detail["no_lists"] is ok + + +def test_format_override_allows_lists() -> None: + text = "- Sales grew.\n- Margins held.\n- Costs fell." + fmt = {"no_lists": False, "bold_headline": False, "max_sentences": None, "single_paragraph": False} + + res = _make_evaluator().evaluate(_item({"format": fmt}), _chat(text)) + + assert "no_lists" not in res.detail + assert "max_sentences" not in res.detail + assert "single_paragraph" not in res.detail + assert res.detail["word_budget"] is True + assert res.passed is True + + +@pytest.mark.parametrize( + ("limit", "check"), + [("max_words", "word_budget"), ("max_headline_words", "headline_words"), ("max_sentences", "max_sentences")], +) +@pytest.mark.parametrize("off", [None, False]) +def test_format_limit_switched_off(limit: str, check: str, off: bool | None) -> None: + res = _make_evaluator().evaluate(_item({"format": {limit: off}}), _chat()) + + assert check not in res.detail + assert res.passed is True + + +@pytest.mark.parametrize( + ("text", "ok"), + [ + (_GOOD, True), + ("**Sales grew.** Driven by 25bdccd8-ef39-40a5-a224-6031d0ab470c.", False), + ("**Sales grew.** Driven by {metric/revenue}.", False), + ("**Sales grew.** Led by label.region.name values.", False), + ("**Sales grew.** The label. Margins held.", True), + ], +) +def test_no_ids(text: str, ok: bool) -> None: + res = _make_evaluator().evaluate(_item(), _chat(text)) + + assert res.detail["no_ids"] is ok + + +@pytest.mark.parametrize( + ("text", "ok"), + [ + (_GOOD, True), + ("**Summary:** Sales grew.", False), + ("**Key takeaways** Sales grew.", False), + ("**Sales grew.** Based on the data, margins held.", False), + ("This chart shows sales growing.", False), + ("**Sales grew.**\nVisualization metadata says margins held.", False), + ("**Sales grew.** The quarterly summary confirms it.", True), + ], +) +def test_no_meta(text: str, ok: bool) -> None: + res = _make_evaluator().evaluate(_item(), _chat(text)) + + assert res.detail["no_meta"] is ok + + +def test_no_data_expected_and_returned_passes_without_judge() -> None: + ev = _make_evaluator() + + res = ev.evaluate(_item({"no_data": True}), _chat(None, no_data=True)) + + assert res.passed is True + assert res.detail["no_data"] is True + assert _judge_calls(ev) == 0 + assert "word_budget" not in res.detail + + +@pytest.mark.parametrize( + ("expected", "text", "no_data"), + [(False, None, True), (True, _GOOD, False), (True, _GOOD, True)], +) +def test_no_data_mismatch_fails(expected: bool, text: str | None, no_data: bool) -> None: + ev = _make_evaluator() + + res = ev.evaluate(_item({"no_data": expected}), _chat(text, no_data=no_data)) + + assert res.passed is False + assert res.detail["no_data"] is False + assert _judge_calls(ev) == 0 + + +def test_deterministic_failure_still_grades_the_judge_criteria() -> None: + ev = _make_evaluator() + + res = ev.evaluate(_item(), _chat("Sales grew and margins held.")) + + assert res.passed is False + assert _judge_calls(ev) == len(DEFAULT_MUST_INCLUDE) + len(DEFAULT_MUST_NOT_INCLUDE) + + +def test_judge_violation_fails_the_item() -> None: + res = _make_evaluator(judge_passes=False).evaluate(_item(), _chat()) + + assert res.passed is False + assert res.detail["default_exclude_exact_figure"] is False + assert res.detail["default_include_one_finding"] is False + + +def test_default_rules_added_unless_disabled() -> None: + ev = _make_evaluator() + res = ev.evaluate(_item({"must_include": ["is about returning customers"]}), _chat()) + + assert ev._positive_judge.score.call_count == len(DEFAULT_MUST_INCLUDE) + 1 + assert ev._violation_judge.score.call_count == len(DEFAULT_MUST_NOT_INCLUDE) + assert res.detail["include_0"] is True + assert all(f"default_exclude_{key}" in res.detail for key in DEFAULT_MUST_NOT_INCLUDE) + + ev = _make_evaluator() + res = ev.evaluate(_item({"default_rules": False, "must_not_include": ["mentions a campaign"]}), _chat()) + + assert _judge_calls(ev) == 1 + assert res.detail["exclude_0"] is True + assert not any(key.startswith("default_") for key in res.detail) + + +def test_item_criteria_are_judged_against_the_takeaway() -> None: + ev = _make_evaluator() + ev.evaluate(_item({"default_rules": False, "rubric": ["plain language"]}), _chat()) + + ev._positive_judge.score.assert_called_once_with( + input="P01 default format, Sales page", expected_output="plain language", actual_output=_GOOD + ) + + +def test_title_number_criterion_text_present() -> None: + criterion = DEFAULT_MUST_NOT_INCLUDE["exact_figure"] + + assert "Top 10" in criterion + assert "in digits" in criterion + assert "'about $270M'" in criterion + assert "'about a third'" in criterion + assert "'roughly doubled'" in criterion + + +def test_calibrated_criterion_wording() -> None: + assert set(DEFAULT_MUST_NOT_INCLUDE) == { + "executive_summary", + "chart_walkthrough", + "background_opener", + "exact_figure", + "date_or_period", + "unshown_cause", + "chart_description", + } + assert "Things to watch" in DEFAULT_MUST_NOT_INCLUDE["executive_summary"] + assert "led by" in DEFAULT_MUST_NOT_INCLUDE["unshown_cause"] + assert set(DEFAULT_MUST_INCLUDE) == {"one_finding"} + + +def test_registered_as_lazy_llm_judge_kind() -> None: + assert "document_takeaway" in supported_test_kinds() + with patch("openai.OpenAI"), patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}): + assert isinstance(get_evaluator("document_takeaway"), DocumentTakeawayEvaluator) + + +def test_no_data_not_stored_as_a_bool_when_a_takeaway_is_expected() -> None: + res = _make_evaluator().evaluate(_item(), _chat()) + + assert not isinstance(res.detail.get("no_data"), bool) + + +def test_unknown_format_key_is_rejected() -> None: + with pytest.raises(ValueError, match="max_sentence"): + _make_evaluator().evaluate(_item({"format": {"max_sentence": 1}}), _chat()) + + +def test_validate_rejects_an_unknown_format_key_without_a_judge_call() -> None: + ev = _make_evaluator() + + with pytest.raises(ValueError, match="'t1'.*max_sentence"): + ev.validate(_item({"format": {"max_sentence": 1}})) + + assert _judge_calls(ev) == 0 + + +def test_validate_rejects_a_format_that_is_not_an_object() -> None: + with pytest.raises(ValueError, match="'t1'.*format must be a JSON object"): + _make_evaluator().validate(_item({"format": ["no_lists"]})) + + +@pytest.mark.parametrize("key", ["must_include", "must_not_include", "rubric"]) +@pytest.mark.parametrize("value", [None, "one criterion", {"a": "b"}]) +def test_validate_rejects_criteria_that_are_not_a_list(key: str, value: Any) -> None: + with pytest.raises(ValueError, match=f"'t1'.*{key} must be a JSON array"): + _make_evaluator().validate(_item({key: value})) + + +def test_validate_accepts_absent_and_empty_criteria() -> None: + ev = _make_evaluator() + ev.validate(_item({})) + ev.validate(_item({"must_include": [], "must_not_include": [], "rubric": []})) + + +def test_validate_accepts_a_valid_item() -> None: + _make_evaluator().validate(_item({"format": {"no_lists": False}, "must_include": ["x"]}))