From 0df433dda9c9e4d89ba33b97d00c9988da66067e Mon Sep 17 00:00:00 2001
From: Roman Rakus <roman.rakus@gooddata.com>
Date: Thu, 22 Oct 2026 10:18:07 +0200
Subject: [PATCH 1/4] feat(gooddata-eval): add takeaway_input dataset field for
 document_takeaway

LX-3167 evaluates AI Publisher takeaways in gd-eval. Dataset items need a
structured input matching the takeaway endpoint body, so the new
TakeawayInput model carries it, Langfuse items read it from input or
metadata, and an item carrying it resolves to the document_takeaway kind.
Unknown keys are rejected so a dataset typo fails loudly.

jira: LX-3167
risk: low
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 .../core/dataset/langfuse_source.py           | 31 +++++-
 .../src/gooddata_eval/core/models.py          | 38 ++++++++
 .../tests/test_langfuse_source.py             | 96 +++++++++++++++++++
 packages/gooddata-eval/tests/test_models.py   | 50 ++++++++++
 4 files changed, 213 insertions(+), 2 deletions(-)

diff --git a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py
index 415cc77f2..0c9047460 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py
@@ -15,9 +15,10 @@
 from typing import Any, TypeVar, cast
 
 import httpx
+from pydantic import ValidationError
 
 from gooddata_eval.core.langfuse._env import make_http_client
-from gooddata_eval.core.models import DatasetItem, SummaryInput
+from gooddata_eval.core.models import DatasetItem, SummaryInput, TakeawayInput
 
 _PAGE_SIZE = 100
 
@@ -59,6 +60,27 @@ def _summary_input_from_raw(raw: dict, expected_output: Any) -> SummaryInput | N
     return SummaryInput.model_validate(candidate) if candidate is not None else None
 
 
+def _takeaway_input_from_raw(raw: dict, expected_output: Any) -> TakeawayInput | None:
+    """Locate a document_takeaway item's `takeaway_input`.
+
+    A value that is not an object raises rather than being skipped: skipping it would
+    drop the item back to the CLI's --kind and judge it as something it was not written as.
+    """
+    for source in (raw.get("input"), raw.get("metadata"), expected_output):
+        if not isinstance(source, dict) or source.get("takeaway_input") is None:
+            continue
+        found = source["takeaway_input"]
+        if not isinstance(found, dict):
+            raise ValueError(
+                f"Langfuse item {raw.get('id')!r}: takeaway_input must be a JSON object, got {type(found).__name__}"
+            )
+        try:
+            return TakeawayInput.model_validate(found)
+        except ValidationError as e:
+            raise ValueError(f"Langfuse item {raw.get('id')!r}: invalid takeaway_input: {e}") from e
+    return None
+
+
 def _user_context_from_raw(raw: dict) -> dict[str, Any] | None:
     """Locate an item's `user_context` -- relayed verbatim as the chat request's `userContext`.
 
@@ -115,7 +137,11 @@ def _item_from_raw(raw: dict, *, dataset_name: str, test_kind: str) -> DatasetIt
     """Map a Langfuse REST API dataset-item dict to a DatasetItem."""
     # REST API returns camelCase: expectedOutput, not expected_output
     expected_output = raw.get("expectedOutput") or raw.get("expected_output")
-    resolved_kind = _infer_test_kind(expected_output, test_kind, raw.get("metadata"))
+    takeaway_input = _takeaway_input_from_raw(raw, expected_output)
+    # An item carrying `takeaway_input` can only be a takeaway item, so it stands in for
+    # the CLI default; an explicit test_kind or a structural match still wins.
+    default_kind = "document_takeaway" if takeaway_input is not None else test_kind
+    resolved_kind = _infer_test_kind(expected_output, default_kind, raw.get("metadata"))
     return DatasetItem(
         id=str(raw["id"]),
         dataset_name=raw.get("datasetName") or dataset_name,
@@ -123,6 +149,7 @@ def _item_from_raw(raw: dict, *, dataset_name: str, test_kind: str) -> DatasetIt
         question=_question_from_input(raw.get("input")),
         expected_output=expected_output,
         summary_input=_summary_input_from_raw(raw, expected_output),
+        takeaway_input=takeaway_input,
         user_context=_user_context_from_raw(raw),
     )
 
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py
index 5bcdd0045..8efba9d28 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/models.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py
@@ -467,6 +467,42 @@ class SummaryInput(BaseModel):
     format_hint: str | None = None
 
 
+class TakeawayVisualization(BaseModel):
+    """A takeaway visualization placed with its own filters."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    visualization_id: str
+    filters: list[dict] = Field(default_factory=list)
+
+
+class TakeawayInstructions(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    document: str | None = None
+    slot: str | None = None
+    format_hint: str | None = None
+
+
+class TakeawayInput(BaseModel):
+    """Structured input for the `document_takeaway` test kind.
+
+    Maps onto the takeaway endpoint's request body
+    (`POST /api/v1/ai/workspaces/{ws}/takeaway`). Authored in snake_case in the
+    dataset; the TakeawayClient maps it to the endpoint's camelCase fields. Unknown
+    keys are rejected at every level so a typo in a dataset item fails loudly instead
+    of being dropped.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    visualizations: list[str | TakeawayVisualization] = Field(min_length=1)
+    filter_context: list[dict] = Field(default_factory=list)
+    instructions: TakeawayInstructions = Field(default_factory=TakeawayInstructions)
+    max_characters: int | None = None
+    agent_id: str | None = None
+
+
 class DatasetItem(BaseModel):
     """Common dataset envelope. `expected_output` stays raw; each evaluator parses its own shape."""
 
@@ -479,6 +515,8 @@ class DatasetItem(BaseModel):
     expected_output: Any
     # Only used by the `dashboard_summary` test kind; ignored by all others.
     summary_input: SummaryInput | None = None
+    # Only used by the `document_takeaway` test kind; ignored by all others.
+    takeaway_input: TakeawayInput | None = None
     # Relayed verbatim as the chat request's `userContext`. Deliberately opaque: gen-ai owns
     # that schema (a discriminated union of view/widget descriptors), so re-modelling it here
     # would only create a second copy to keep in sync.
diff --git a/packages/gooddata-eval/tests/test_langfuse_source.py b/packages/gooddata-eval/tests/test_langfuse_source.py
index 92448b01e..028d7d391 100644
--- a/packages/gooddata-eval/tests/test_langfuse_source.py
+++ b/packages/gooddata-eval/tests/test_langfuse_source.py
@@ -73,6 +73,102 @@ def test_item_from_raw_summary_input_absent_is_none():
     assert item.summary_input is None
 
 
+def test_langfuse_item_takeaway_input_from_input_object() -> None:
+    raw = {
+        "id": "lf-tk-1",
+        "datasetName": "ds",
+        "input": {"question": "Takeaway for the page", "takeaway_input": {"visualizations": ["v1"]}},
+        "metadata": {"takeaway_input": {"visualizations": ["from-metadata"]}},
+        "expectedOutput": {"test_kind": "document_takeaway"},
+    }
+    item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization")
+    assert item.takeaway_input is not None
+    assert item.takeaway_input.visualizations == ["v1"]
+
+
+def test_langfuse_item_takeaway_input_from_metadata() -> None:
+    raw = {
+        "id": "lf-tk-2",
+        "datasetName": "ds",
+        "input": "Takeaway for the page",
+        "metadata": {"takeaway_input": {"visualizations": ["v2"], "max_characters": 1000}},
+        "expectedOutput": {"must_include": ["revenue"]},
+    }
+    item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization")
+    assert item.test_kind == "document_takeaway"
+    assert item.takeaway_input.visualizations == ["v2"]
+    assert item.takeaway_input.max_characters == 1000
+
+
+def test_langfuse_item_takeaway_input_from_expected_output() -> None:
+    raw = {
+        "id": "lf-tk-5",
+        "datasetName": "ds",
+        "input": "Takeaway for the page",
+        "expectedOutput": {"takeaway_input": {"visualizations": ["v3"]}},
+    }
+    item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization")
+    assert item.test_kind == "document_takeaway"
+    assert item.takeaway_input.visualizations == ["v3"]
+
+
+def test_langfuse_item_structural_kind_beats_takeaway_input() -> None:
+    raw = {
+        "id": "lf-tk-6",
+        "datasetName": "ds",
+        "input": {"question": "Show revenue", "takeaway_input": {"visualizations": ["v1"]}},
+        "expectedOutput": {"visualization": {"id": "x", "type": "", "query": {"fields": {}}}},
+    }
+    item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization")
+    assert item.test_kind == "vis_agentic"
+
+
+def test_langfuse_item_invalid_takeaway_input_names_the_item() -> None:
+    raw = {
+        "id": "lf-tk-bad",
+        "datasetName": "ds",
+        "input": {"question": "Takeaway", "takeaway_input": {"visualizations": ["v1"], "max_chars": 400}},
+        "expectedOutput": {},
+    }
+    with pytest.raises(ValueError, match=r"(?s)'lf-tk-bad'.*max_chars"):
+        _item_from_raw(raw, dataset_name="ds", test_kind="visualization")
+
+
+@pytest.mark.parametrize("takeaway_input", ["v1", ["v1"]])
+def test_langfuse_item_non_object_takeaway_input_names_the_item(takeaway_input: object) -> None:
+    raw = {
+        "id": "lf-tk-shape",
+        "datasetName": "ds",
+        "input": "Takeaway",
+        "metadata": {"takeaway_input": takeaway_input},
+        "expectedOutput": {},
+    }
+    with pytest.raises(ValueError, match="'lf-tk-shape'.*takeaway_input"):
+        _item_from_raw(raw, dataset_name="ds", test_kind="visualization")
+
+
+def test_langfuse_item_infers_document_takeaway_kind() -> None:
+    raw = {
+        "id": "lf-tk-3",
+        "datasetName": "ds",
+        "input": {"question": "Takeaway for the page", "takeaway_input": {"visualizations": ["v1"]}},
+        "expectedOutput": {"no_data": False, "must_include": ["revenue"]},
+    }
+    item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization")
+    assert item.test_kind == "document_takeaway"
+
+
+def test_langfuse_item_explicit_kind_beats_takeaway_input() -> None:
+    raw = {
+        "id": "lf-tk-4",
+        "datasetName": "ds",
+        "input": {"question": "Takeaway for the page", "takeaway_input": {"visualizations": ["v1"]}},
+        "expectedOutput": {"test_kind": "agentic_general_question"},
+    }
+    item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization")
+    assert item.test_kind == "agentic_general_question"
+
+
 def test_load_langfuse_dataset_calls_rest_api(monkeypatch):
     monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test")
     monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test")
diff --git a/packages/gooddata-eval/tests/test_models.py b/packages/gooddata-eval/tests/test_models.py
index 0c09a23ef..a34cc2a58 100644
--- a/packages/gooddata-eval/tests/test_models.py
+++ b/packages/gooddata-eval/tests/test_models.py
@@ -1,9 +1,12 @@
 # (C) 2026 GoodData Corporation
+import pytest
 from gooddata_eval.core.models import (
     ChatResult,
     CreatedVisualization,
     DatasetItem,
     ReasoningStepEvent,
+    TakeawayInput,
+    TakeawayVisualization,
     ToolCallEvent,
     build_tool_calls,
     timeline_detail,
@@ -171,3 +174,50 @@ def test_a_visualization_without_an_id_still_parses():
     """The agent sometimes omits `id`; nothing scores on it, so it must not error the item."""
     viz = CreatedVisualization.model_validate({"type": "bar_chart", "query": {"fields": {"m": "metric/x"}}})
     assert viz.id is None
+
+
+def test_takeaway_input_accepts_id_and_placement_visualizations() -> None:
+    takeaway_input = TakeawayInput.model_validate(
+        {
+            "visualizations": ["v1", {"visualization_id": "v2", "filters": [{"positiveAttributeFilter": {}}]}],
+            "instructions": {"slot": "page", "format_hint": "one sentence"},
+            "max_characters": 400,
+        }
+    )
+    assert takeaway_input.visualizations == [
+        "v1",
+        TakeawayVisualization(visualization_id="v2", filters=[{"positiveAttributeFilter": {}}]),
+    ]
+    assert takeaway_input.filter_context == []
+    assert takeaway_input.instructions.format_hint == "one sentence"
+    assert takeaway_input.max_characters == 400
+    assert takeaway_input.agent_id is None
+
+
+def test_takeaway_input_rejects_unknown_key() -> None:
+    with pytest.raises(ValueError, match="max_chars"):
+        TakeawayInput.model_validate({"visualizations": ["v1"], "max_chars": 400})
+
+
+@pytest.mark.parametrize(
+    "raw",
+    [
+        {"visualizations": [{"visualization_id": "v1", "filter": []}]},
+        {"visualizations": ["v1"], "instructions": {"formatHint": "one sentence"}},
+    ],
+)
+def test_takeaway_input_rejects_unknown_nested_key(raw: dict) -> None:
+    with pytest.raises(ValueError, match="Extra inputs are not permitted"):
+        TakeawayInput.model_validate(raw)
+
+
+def test_takeaway_input_requires_a_visualization() -> None:
+    with pytest.raises(ValueError, match="visualizations"):
+        TakeawayInput.model_validate({"visualizations": []})
+
+
+def test_dataset_item_takeaway_input_defaults_to_none() -> None:
+    item = DatasetItem.model_validate(
+        {"id": "q1", "dataset_name": "d1", "test_kind": "dashboard_summary", "question": "q", "expected_output": {}}
+    )
+    assert item.takeaway_input is None

From 1b59168385a07d74763a153326084316697804cb Mon Sep 17 00:00:00 2001
From: Roman Rakus <roman.rakus@gooddata.com>
Date: Thu, 22 Oct 2026 13:35:55 +0200
Subject: [PATCH 2/4] feat(gooddata-eval): add TakeawayClient for the takeaway
 endpoint

The document_takeaway kind (LX-3167) needs a client for the AI
Publisher takeaway endpoint. It maps TakeawayInput to the camelCase
body, omitting every unset field because the server DTO forbids extra
keys and nulls, and always sends filterContext.

The whole response is kept on a new optional ChatResult.raw_response so
the evaluator can tell noData apart from an empty takeaway; every other
kind leaves it None. A non-2xx answer raises HTTPStatusError with the
status and response body in its message, so an item error shows why the
endpoint refused (flag off, no agent, failed visualization).

jira: LX-3167
risk: low
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 packages/gooddata-eval/AGENTS.md              |  19 +-
 .../src/gooddata_eval/core/_retry.py          |  71 +++++
 .../src/gooddata_eval/core/chat/sse_client.py |  72 +----
 .../src/gooddata_eval/core/models.py          |   3 +
 .../gooddata_eval/core/takeaway/__init__.py   |   1 +
 .../core/takeaway/http_client.py              | 104 +++++++
 .../gooddata-eval/tests/test_sse_client.py    |   6 +
 .../tests/test_takeaway_client.py             | 257 ++++++++++++++++++
 8 files changed, 466 insertions(+), 67 deletions(-)
 create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/_retry.py
 create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/takeaway/__init__.py
 create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/takeaway/http_client.py
 create mode 100644 packages/gooddata-eval/tests/test_takeaway_client.py

diff --git a/packages/gooddata-eval/AGENTS.md b/packages/gooddata-eval/AGENTS.md
index 3769cfafb..3c7846a5c 100644
--- a/packages/gooddata-eval/AGENTS.md
+++ b/packages/gooddata-eval/AGENTS.md
@@ -155,8 +155,8 @@ Run this before guessing a `--model` string.
 | `GD_EVAL_JUDGE_MODEL` | judge model, same as `--judge-model` |
 | `GD_EVAL_AGENT_ID` | which agent to drive, same as `--agent-id` |
 | `LANGFUSE_HOST`, `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` | `--langfuse`, `--langfuse-dataset` |
-| `GOODDATA_EVAL_CHAT_*` | SSE retry, backoff and timeout knobs |
-| `GOODDATA_EVAL_TRACE_LABELS` | `key=value,...` labels stamped on every observation of a chat conversation via W3C baggage; `model_version` also sets the observation `version`, so set it only for a one-model run: every `--model` in one process gets the same labels. Keys must match `[A-Za-z0-9_-]+`; others are skipped with a warning. Dashboard-summary runs are not labelled |
+| `GOODDATA_EVAL_CHAT_*` | retry and backoff knobs shared by the chat and takeaway clients (`core/_retry.py`), and the SSE turn and item timeouts |
+| `GOODDATA_EVAL_TRACE_LABELS` | `key=value,...` labels stamped on every observation of a chat conversation via W3C baggage; `model_version` also sets the observation `version`, so set it only for a one-model run: every `--model` in one process gets the same labels. Keys must match `[A-Za-z0-9_-]+`; others are skipped with a warning. Dashboard-summary and document-takeaway runs are not labelled |
 | `GOODDATA_EVAL_JOIN_GENAI_TRACE` | on unless set to `0`, `false`, `no`, `off` or empty. An agentic item's chat turns ask gen-ai to open them under gd-eval's experiment root, so each item is one Langfuse trace with its scores written once. A gen-ai that drops caller baggage keeps today's separate traces; see the gotcha below |
 | `GD_EVAL_TIMERS` | same as `--timers` |
 
@@ -181,12 +181,15 @@ That default is correct: agent tool calls create real server-side objects (metri
 Adding a kind to `PARALLEL_SAFE_TEST_KINDS` is a deliberate assertion that it is read-only,
 which nothing in the package can prove for you.
 
-**The SSE client's retry predicate is load-bearing.** `core/chat/sse_client.py` retries
-429/502/503/504 and `httpx.RemoteProtocolError` (a mid-stream disconnect) with exponential
-backoff, and treats a `METADATA_SYNC_IN_PROGRESS` payload as transient. The
-`RemoteProtocolError` case was added after it was confirmed live to contaminate a small
-percentage of visualization runs with a hard fail and no retry. Narrowing that predicate
-reintroduces the problem.
+**The SSE client's retry predicate is load-bearing.** The backoff loop and the retryable
+status set (429/500/502/503/504) live in `core/_retry.py`, shared by the chat and takeaway
+clients and tuned by the same `GOODDATA_EVAL_CHAT_*` knobs. `core/chat/sse_client.py`
+retries that status set and `httpx.RemoteProtocolError` (a mid-stream disconnect), and
+treats a `METADATA_SYNC_IN_PROGRESS` payload as transient. The `RemoteProtocolError` case
+was added after it was confirmed live to contaminate a small percentage of visualization
+runs with a hard fail and no retry. Narrowing that predicate reintroduces the problem. The
+takeaway client retries the same status set plus connection failures, but not a read
+timeout: a slow page would spend the whole timeout again on every attempt.
 
 **Langfuse trace linking is deliberately off the item critical path.** Polling for trace
 ingestion has no pass/fail signal and inflates or misattributes per-item latency, so
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/_retry.py b/packages/gooddata-eval/src/gooddata_eval/core/_retry.py
new file mode 100644
index 000000000..982f8cb74
--- /dev/null
+++ b/packages/gooddata-eval/src/gooddata_eval/core/_retry.py
@@ -0,0 +1,71 @@
+# (C) 2026 GoodData Corporation
+"""Bounded exponential-backoff retry shared by the gen-ai HTTP clients.
+
+The budget is read from the ``GOODDATA_EVAL_CHAT_*`` environment on every call, so the chat
+and takeaway clients retune together.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import time
+from collections.abc import Callable
+from typing import TypeVar
+
+_log = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+# 500 is here on evidence, not on principle: in one visualization eval batch it hard-failed
+# 10 of 56 runs with zero retry attempts, and every affected question scored normally when the
+# same question/model ran again the next day -- i.e. transient gen-ai faults, not deterministic
+# server bugs. A genuinely deterministic 500 still terminates, just after the bounded backoff.
+RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({429, 500, 502, 503, 504})
+
+
+def int_env(name: str, default: int) -> int:
+    """Read an int from the environment, falling back to ``default`` when unset or blank."""
+    raw = os.getenv(name)
+    return int(raw) if raw else default
+
+
+def float_env(name: str, default: float) -> float:
+    """Read a float from the environment, falling back to ``default`` when unset or blank."""
+    raw = os.getenv(name)
+    return float(raw) if raw else default
+
+
+# Retry budget defaults, giving a ~2 min worst-case cap per send (5/10/20/40/60s).
+# Each is overridable via env so CI can retune without cutting a new release --
+# read per call rather than at import, so an exported value cannot silently
+# rewrite what a test that patches these attributes expects.
+_MAX_RETRIES_DEFAULT = 5
+_INITIAL_BACKOFF_S_DEFAULT = 5.0
+_BACKOFF_FACTOR_DEFAULT = 2.0
+_MAX_BACKOFF_S_DEFAULT = 60.0
+
+
+def retry_transient(operation: Callable[[], T], *, is_retryable: Callable[[Exception], bool]) -> T:
+    """Run ``operation``; retry retryable failures with bounded exponential backoff."""
+    max_retries = int_env("GOODDATA_EVAL_CHAT_MAX_RETRIES", _MAX_RETRIES_DEFAULT)
+    delay = float_env("GOODDATA_EVAL_CHAT_INITIAL_BACKOFF_S", _INITIAL_BACKOFF_S_DEFAULT)
+    factor = float_env("GOODDATA_EVAL_CHAT_BACKOFF_FACTOR", _BACKOFF_FACTOR_DEFAULT)
+    max_backoff = float_env("GOODDATA_EVAL_CHAT_MAX_BACKOFF_S", _MAX_BACKOFF_S_DEFAULT)
+    for attempt in range(max_retries + 1):  # 0..N => N retries + 1 initial attempt
+        try:
+            return operation()
+        except Exception as exc:  # noqa: PERF203 — retry loop: per-attempt try/except is intentional
+            if attempt == max_retries or not is_retryable(exc):
+                raise
+            sleep_s = min(delay, max_backoff)
+            _log.warning(
+                "Transient gen-ai error (attempt %d/%d): %s; retrying in %.0fs",
+                attempt + 1,
+                max_retries + 1,
+                exc,
+                sleep_s,
+            )
+            time.sleep(sleep_s)
+            delay *= factor
+    raise AssertionError("unreachable")  # loop either returns or raises
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py
index ad58382e0..f5e77b2d0 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py
@@ -20,11 +20,12 @@
 import re
 import time
 from dataclasses import dataclass, field
-from typing import Any, Callable, Iterable, TypeVar
+from typing import Any, Iterable
 from urllib.parse import quote
 
 import httpx
 
+from gooddata_eval.core._retry import RETRYABLE_STATUS_CODES, float_env, int_env, retry_transient
 from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort
 from gooddata_eval.core.langfuse.item_scope import JoinedRun, baggage_entries, current_scope, mark_joined, trace_ids_for
 from gooddata_eval.core.models import ChatResult, DatasetItem
@@ -32,6 +33,13 @@
 
 _log = logging.getLogger(__name__)
 
+# The retry helpers live in core/_retry.py, shared with the takeaway client. The private
+# names stay importable from here: gdc-nas tavern-e2e imports them from this module.
+_int_env = int_env
+_float_env = float_env
+_RETRYABLE_STATUS_CODES = RETRYABLE_STATUS_CODES
+_retry_transient = retry_transient
+
 SSE_DATA_PREFIX = "data: "
 SSE_EVENT_PREFIX = "event: "
 # gen-ai's last event, only if at least one item was already emitted (conversations_controller.py).
@@ -39,11 +47,6 @@
 # gen-ai's first event of a turn; its data carries the turn's responseId and traceId.
 _RESPONSE_STARTED_EVENT = "response_started"
 
-# 500 is here on evidence, not on principle: in one visualization eval batch it hard-failed
-# 10 of 56 runs with zero retry attempts, and every affected question scored normally when the
-# same question/model ran again the next day -- i.e. transient gen-ai faults, not deterministic
-# server bugs. A genuinely deterministic 500 still terminates, just after the bounded backoff.
-_RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({429, 500, 502, 503, 504})
 _METADATA_SYNC_MARKER = "METADATA_SYNC_IN_PROGRESS"
 # `reason` values gen-ai puts on the error event of a turn that finished without an answer.
 _TURN_INCOMPLETE_REASONS: frozenset[str] = frozenset({"max_iterations", "max_tokens", "content_filter"})
@@ -112,27 +115,6 @@ class TurnTimeoutError(ChatError):
     retrying it spends the budget again."""
 
 
-def _int_env(name: str, default: int) -> int:
-    """Read an int from the environment, falling back to ``default`` when unset or blank."""
-    raw = os.getenv(name)
-    return int(raw) if raw else default
-
-
-def _float_env(name: str, default: float) -> float:
-    """Read a float from the environment, falling back to ``default`` when unset or blank."""
-    raw = os.getenv(name)
-    return float(raw) if raw else default
-
-
-# Retry budget defaults, giving a ~2 min worst-case cap per send (5/10/20/40/60s).
-# Each is overridable via env so CI can retune without cutting a new release --
-# read per call rather than at import, so an exported value cannot silently
-# rewrite what a test that patches these attributes expects.
-_MAX_RETRIES_DEFAULT = 5
-_INITIAL_BACKOFF_S_DEFAULT = 5.0
-_BACKOFF_FACTOR_DEFAULT = 2.0
-_MAX_BACKOFF_S_DEFAULT = 60.0
-
 # Wall-clock cap on a single agent turn, 0 = uncapped. httpx's `timeout` is per-read,
 # so an agent that keeps emitting reasoning events can stream for many minutes without
 # ever tripping it -- this is what bounds a runaway item and lets the run move on.
@@ -199,14 +181,11 @@ def set_default_item_timeout(seconds: float | None) -> None:
     _ITEM_TIMEOUT_S = seconds or 0.0
 
 
-T = TypeVar("T")
-
-
 def _is_retryable_exc(exc: Exception) -> bool:
     if isinstance(exc, TransientChatError):
         return True
     if isinstance(exc, httpx.HTTPStatusError):
-        return exc.response.status_code in _RETRYABLE_STATUS_CODES
+        return exc.response.status_code in RETRYABLE_STATUS_CODES
     # Mid-stream disconnect ("peer closed connection without sending complete
     # message body") -- pure network flake, not a real agent/content failure.
     # Confirmed live: contaminated ~1-4% of visualization runs with a hard
@@ -214,31 +193,6 @@ def _is_retryable_exc(exc: Exception) -> bool:
     return isinstance(exc, httpx.RemoteProtocolError)
 
 
-def _retry_transient(operation: Callable[[], T], *, is_retryable: Callable[[Exception], bool]) -> T:
-    """Run ``operation``; retry retryable failures with bounded exponential backoff."""
-    max_retries = _int_env("GOODDATA_EVAL_CHAT_MAX_RETRIES", _MAX_RETRIES_DEFAULT)
-    delay = _float_env("GOODDATA_EVAL_CHAT_INITIAL_BACKOFF_S", _INITIAL_BACKOFF_S_DEFAULT)
-    factor = _float_env("GOODDATA_EVAL_CHAT_BACKOFF_FACTOR", _BACKOFF_FACTOR_DEFAULT)
-    max_backoff = _float_env("GOODDATA_EVAL_CHAT_MAX_BACKOFF_S", _MAX_BACKOFF_S_DEFAULT)
-    for attempt in range(max_retries + 1):  # 0..N => N retries + 1 initial attempt
-        try:
-            return operation()
-        except Exception as exc:  # noqa: PERF203 — retry loop: per-attempt try/except is intentional
-            if attempt == max_retries or not is_retryable(exc):
-                raise
-            sleep_s = min(delay, max_backoff)
-            _log.warning(
-                "Transient gen-ai error (attempt %d/%d): %s; retrying in %.0fs",
-                attempt + 1,
-                max_retries + 1,
-                exc,
-                sleep_s,
-            )
-            time.sleep(sleep_s)
-            delay *= factor
-    raise AssertionError("unreachable")  # loop either returns or raises
-
-
 @dataclass
 class _SseAccumulator:
     text_parts: list[str] = field(default_factory=list)
@@ -456,7 +410,7 @@ def parse_sse_lines(lines: Iterable[str]) -> ChatResult:
                 raise TurnIncompleteError(
                     message, status_code=code, detail=detail, partial_result=_build_chat_result(acc), reason=reason
                 )
-            if code in _RETRYABLE_STATUS_CODES:
+            if code in RETRYABLE_STATUS_CODES:
                 raise TransientChatError(
                     message, status_code=code, detail=detail, partial_result=_build_chat_result(acc), reason=reason
                 )
@@ -557,7 +511,7 @@ def _do() -> str:
 
         # NOTE: retrying create is not idempotent — a created-then-503 can leak an
         # orphaned (ephemeral) conversation. Acceptable for eval; do not reuse blindly.
-        conversation_id = _retry_transient(_do, is_retryable=_is_retryable_exc)
+        conversation_id = retry_transient(_do, is_retryable=_is_retryable_exc)
         # Anchor the per-item clock here: for an agentic item this conversation carries
         # every turn, so the budget must run from its creation, not from each send.
         self._conversation_started = time.monotonic()
@@ -603,7 +557,7 @@ def _do() -> ChatResult:
                 return result
 
         try:
-            result = _retry_transient(_do, is_retryable=_is_retryable_exc)
+            result = retry_transient(_do, is_retryable=_is_retryable_exc)
         except ChatError as exc:
             if joined is not None and exc.partial_result is not None:
                 self._record_join(conversation_id, exc.partial_result.trace_id, joined)
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py
index 8efba9d28..d71f34d93 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/models.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py
@@ -347,6 +347,9 @@ class ChatResult(BaseModel):
     stream_ended: bool = False
     # Wall-clock seconds for the whole chat turn, timed by the client.
     turn_wall_clock_sec: float | None = None
+    # The endpoint's whole JSON response, for single-shot endpoints whose answer carries
+    # more than its text (the takeaway endpoint's `noData`). None for chat responses.
+    raw_response: dict[str, Any] | None = None
 
 
 def shift_and_index_events(
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/takeaway/__init__.py b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/__init__.py
new file mode 100644
index 000000000..efe7c60c8
--- /dev/null
+++ b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/__init__.py
@@ -0,0 +1 @@
+# (C) 2026 GoodData Corporation
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/takeaway/http_client.py b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/http_client.py
new file mode 100644
index 000000000..d2ef8d9e5
--- /dev/null
+++ b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/http_client.py
@@ -0,0 +1,104 @@
+# (C) 2026 GoodData Corporation
+"""HTTP client for the AI Publisher takeaway endpoint.
+
+Like the dashboard-summary endpoint, it executes each visualization server-side and
+returns plain synchronous JSON. The response is adapted into a ``ChatResult``
+(takeaway text -> ``text_response``); the whole response body is kept on
+``raw_response`` so the evaluator can tell a ``noData`` answer from an empty takeaway.
+
+Endpoint (gen-ai service):
+    POST /api/v1/ai/workspaces/{workspace_id}/takeaway
+
+The request DTO rejects unknown keys and nulls are not sent, so every unset optional
+field is omitted rather than sent as ``null``.
+"""
+
+from typing import Any
+
+import httpx
+
+from gooddata_eval.core._retry import RETRYABLE_STATUS_CODES, retry_transient
+from gooddata_eval.core.models import ChatResult, DatasetItem, TakeawayInput, TakeawayVisualization
+
+_PATH = "/api/v1/ai/workspaces/{workspace_id}/takeaway"
+
+
+def _visualization_body(visualization: str | TakeawayVisualization) -> str | dict[str, Any]:
+    if isinstance(visualization, str):
+        return visualization
+    return {"visualizationId": visualization.visualization_id, "filters": visualization.filters}
+
+
+# Failures that say nothing about the request. A read timeout is left out: the page was
+# being computed, and retrying spends the whole timeout again on every attempt.
+_RETRYABLE_TRANSPORT_ERRORS = (httpx.ConnectError, httpx.ConnectTimeout, httpx.RemoteProtocolError, httpx.PoolTimeout)
+
+
+def _is_retryable(exc: Exception) -> bool:
+    if isinstance(exc, httpx.HTTPStatusError):
+        return exc.response.status_code in RETRYABLE_STATUS_CODES
+    return isinstance(exc, _RETRYABLE_TRANSPORT_ERRORS)
+
+
+def _request_body(ti: TakeawayInput) -> dict[str, Any]:
+    body: dict[str, Any] = {}
+    if ti.agent_id is not None:
+        body["agentId"] = ti.agent_id
+    instructions = {
+        key: value
+        for key, value in (
+            ("document", ti.instructions.document),
+            ("slot", ti.instructions.slot),
+            ("formatHint", ti.instructions.format_hint),
+        )
+        if value is not None
+    }
+    if instructions:
+        body["instructions"] = instructions
+    if ti.max_characters is not None:
+        body["maxCharacters"] = ti.max_characters
+    body["data"] = {
+        "visualizations": [_visualization_body(v) for v in ti.visualizations],
+        "filterContext": ti.filter_context,
+    }
+    return body
+
+
+class TakeawayClient:
+    """Single-shot client for the takeaway endpoint."""
+
+    def __init__(self, host: str, token: str, workspace_id: str, *, timeout: float = 120.0):
+        self._url = f"{host.rstrip('/')}{_PATH.format(workspace_id=workspace_id)}"
+        self._auth = {"Authorization": f"Bearer {token}"}
+        self._client = httpx.Client(timeout=timeout)
+
+    def ask(self, item: DatasetItem) -> ChatResult:
+        """Request a takeaway for one dataset item and adapt it to a ChatResult.
+
+        Raises:
+            ValueError: The item carries no `takeaway_input`.
+            httpx.HTTPStatusError: The endpoint answered non-2xx; the message carries the
+                status code and the response body. Transient statuses and connection
+                failures are retried first, with the chat client's backoff and budget.
+        """
+        ti = item.takeaway_input
+        if ti is None:
+            raise ValueError(f"document_takeaway item '{item.id}' is missing required 'takeaway_input'.")
+
+        body = _request_body(ti)
+
+        def _post() -> httpx.Response:
+            resp = self._client.post(self._url, json=body, headers={**self._auth, "Content-Type": "application/json"})
+            if not resp.is_success:
+                raise httpx.HTTPStatusError(
+                    f"Takeaway request failed with {resp.status_code} for {self._url}: {resp.text}",
+                    request=resp.request,
+                    response=resp,
+                )
+            return resp
+
+        data = retry_transient(_post, is_retryable=_is_retryable).json()
+        return ChatResult(text_response=data.get("takeaway") or "", raw_response=data)
+
+    def close(self) -> None:
+        self._client.close()
diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py
index 79f9cb818..53982fbf3 100644
--- a/packages/gooddata-eval/tests/test_sse_client.py
+++ b/packages/gooddata-eval/tests/test_sse_client.py
@@ -6,6 +6,7 @@
 
 import httpx
 import pytest
+from gooddata_eval.core import _retry as retry_mod
 from gooddata_eval.core.chat import sse_client as sse_mod
 from gooddata_eval.core.chat.sse_client import (
     ChatClient,
@@ -1350,3 +1351,8 @@ def test_a_turn_that_fails_after_response_started_still_counts_as_joined(monkeyp
         client.send_message("conv-failed-turn", "q")
 
     assert joined_turns("conv-failed-turn") == 1
+
+
+def test_retry_names_external_callers_import_stay_on_sse_client() -> None:
+    assert sse_mod._RETRYABLE_STATUS_CODES is retry_mod.RETRYABLE_STATUS_CODES
+    assert sse_mod._retry_transient is retry_mod.retry_transient
diff --git a/packages/gooddata-eval/tests/test_takeaway_client.py b/packages/gooddata-eval/tests/test_takeaway_client.py
new file mode 100644
index 000000000..5fe7b084d
--- /dev/null
+++ b/packages/gooddata-eval/tests/test_takeaway_client.py
@@ -0,0 +1,257 @@
+# (C) 2026 GoodData Corporation
+import json
+from collections.abc import Callable
+
+import httpx
+import pytest
+from gooddata_eval.core import _retry as retry_mod
+from gooddata_eval.core.models import DatasetItem, TakeawayInput
+from gooddata_eval.core.takeaway.http_client import TakeawayClient
+
+_TAKEAWAY_RESPONSE = {
+    "takeaway": "**Revenue grew 12%.** Driven by the West region.",
+    "noData": False,
+    "generatedAt": "2026-10-22T10:00:00Z",
+    "agentId": "agent-1",
+    "contextOmitted": [],
+}
+
+
+def _item(takeaway_input: dict | None) -> DatasetItem:
+    return DatasetItem(
+        id="t1",
+        dataset_name="d",
+        test_kind="document_takeaway",
+        question="Takeaway for the page",
+        expected_output={},
+        takeaway_input=TakeawayInput.model_validate(takeaway_input) if takeaway_input is not None else None,
+    )
+
+
+def _client(handler: Callable[[httpx.Request], httpx.Response]) -> TakeawayClient:
+    client = TakeawayClient(host="https://h/", token="tok", workspace_id="ws")
+    client._client = httpx.Client(transport=httpx.MockTransport(handler))
+    return client
+
+
+def _capturing(captured: dict, response: dict | None = None) -> Callable[[httpx.Request], httpx.Response]:
+    def handler(request: httpx.Request) -> httpx.Response:
+        captured["url"] = str(request.url)
+        captured["auth"] = request.headers.get("authorization")
+        captured["body"] = json.loads(request.content)
+        return httpx.Response(200, json=response or _TAKEAWAY_RESPONSE)
+
+    return handler
+
+
+def test_body_is_camel_case_and_omits_unset() -> None:
+    captured: dict = {}
+    client = _client(_capturing(captured))
+
+    result = client.ask(
+        _item(
+            {
+                "visualizations": ["v1"],
+                "filter_context": [{"dateFilter": {}}],
+                "instructions": {"slot": "page", "format_hint": "one sentence"},
+                "max_characters": 1000,
+                "agent_id": "agent-1",
+            }
+        )
+    )
+
+    assert captured["url"] == "https://h/api/v1/ai/workspaces/ws/takeaway"
+    assert captured["auth"] == "Bearer tok"
+    assert captured["body"] == {
+        "agentId": "agent-1",
+        "instructions": {"slot": "page", "formatHint": "one sentence"},
+        "maxCharacters": 1000,
+        "data": {"visualizations": ["v1"], "filterContext": [{"dateFilter": {}}]},
+    }
+    assert result.text_response == "**Revenue grew 12%.** Driven by the West region."
+
+
+def test_placement_visualization_serialized_with_filters() -> None:
+    captured: dict = {}
+    client = _client(_capturing(captured))
+
+    client.ask(
+        _item(
+            {
+                "visualizations": [
+                    "v1",
+                    {"visualization_id": "v2", "filters": [{"positiveAttributeFilter": {}}]},
+                    {"visualization_id": "v3"},
+                ]
+            }
+        )
+    )
+
+    assert captured["body"] == {
+        "data": {
+            "visualizations": [
+                "v1",
+                {"visualizationId": "v2", "filters": [{"positiveAttributeFilter": {}}]},
+                {"visualizationId": "v3", "filters": []},
+            ],
+            "filterContext": [],
+        },
+    }
+
+
+def test_instructions_omitted_when_empty() -> None:
+    captured: dict = {}
+    client = _client(_capturing(captured))
+
+    client.ask(_item({"visualizations": ["v1"], "instructions": {}}))
+
+    assert "instructions" not in captured["body"]
+    assert captured["body"]["data"]["filterContext"] == []
+
+
+def test_no_data_response_kept_as_no_data_not_empty_text() -> None:
+    no_data = {**_TAKEAWAY_RESPONSE, "takeaway": None, "noData": True, "contextOmitted": ["v1"]}
+    client = _client(_capturing({}, no_data))
+
+    result = client.ask(_item({"visualizations": ["v1"]}))
+
+    assert result.text_response == ""
+    assert result.raw_response is not None
+    assert result.raw_response["noData"] is True
+    assert result.raw_response["takeaway"] is None
+    assert result.raw_response["agentId"] == "agent-1"
+    assert result.raw_response["contextOmitted"] == ["v1"]
+
+
+def test_http_error_carries_status_and_body(sleeps: list[float]) -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(502, json={"detail": "Visualizations ['v1'] failed to execute."})
+
+    client = _client(handler)
+
+    with pytest.raises(httpx.HTTPStatusError) as exc_info:
+        client.ask(_item({"visualizations": ["v1"]}))
+
+    assert exc_info.value.response.status_code == 502
+    assert "502" in str(exc_info.value)
+    assert "failed to execute" in str(exc_info.value)
+
+
+def test_redirect_is_an_http_error() -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(302, headers={"Location": "https://h/login"}, text="Found")
+
+    client = _client(handler)
+
+    with pytest.raises(httpx.HTTPStatusError, match="302"):
+        client.ask(_item({"visualizations": ["v1"]}))
+
+
+def test_item_without_takeaway_input_names_the_item() -> None:
+    client = TakeawayClient(host="https://h", token="tok", workspace_id="ws")
+
+    with pytest.raises(ValueError, match="'t1'.*takeaway_input"):
+        client.ask(_item(None))
+
+
+@pytest.fixture
+def sleeps(monkeypatch: pytest.MonkeyPatch) -> list[float]:
+    for name in (
+        "GOODDATA_EVAL_CHAT_MAX_RETRIES",
+        "GOODDATA_EVAL_CHAT_INITIAL_BACKOFF_S",
+        "GOODDATA_EVAL_CHAT_BACKOFF_FACTOR",
+        "GOODDATA_EVAL_CHAT_MAX_BACKOFF_S",
+    ):
+        monkeypatch.delenv(name, raising=False)
+    recorded: list[float] = []
+    monkeypatch.setattr(retry_mod.time, "sleep", recorded.append)
+    return recorded
+
+
+def _scripted(*responses: httpx.Response | Exception) -> tuple[Callable[[httpx.Request], httpx.Response], list[int]]:
+    calls: list[int] = []
+    it = iter(responses)
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls.append(1)
+        nxt = next(it)
+        if isinstance(nxt, Exception):
+            raise nxt
+        return nxt
+
+    return handler, calls
+
+
+@pytest.mark.parametrize("status", [429, 502, 503, 504])
+def test_transient_status_is_retried(status: int, sleeps: list[float]) -> None:
+    handler, calls = _scripted(httpx.Response(status, text="busy"), httpx.Response(200, json=_TAKEAWAY_RESPONSE))
+
+    result = _client(handler).ask(_item({"visualizations": ["v1"]}))
+
+    assert result.text_response == _TAKEAWAY_RESPONSE["takeaway"]
+    assert len(calls) == 2
+    assert sleeps == [5.0]
+
+
+@pytest.mark.parametrize(
+    "error",
+    [
+        httpx.ConnectError("refused"),
+        httpx.ConnectTimeout("connect timed out"),
+        httpx.RemoteProtocolError("peer closed connection"),
+        httpx.PoolTimeout("pool exhausted"),
+    ],
+)
+def test_transient_transport_error_is_retried(error: httpx.TransportError, sleeps: list[float]) -> None:
+    handler, calls = _scripted(error, httpx.Response(200, json=_TAKEAWAY_RESPONSE))
+
+    _client(handler).ask(_item({"visualizations": ["v1"]}))
+
+    assert len(calls) == 2
+
+
+@pytest.mark.parametrize(
+    "error",
+    [
+        httpx.ReadTimeout("read timed out"),
+        httpx.WriteTimeout("write timed out"),
+        httpx.UnsupportedProtocol("no scheme"),
+    ],
+)
+def test_slow_or_misconfigured_request_is_not_retried(error: httpx.TransportError, sleeps: list[float]) -> None:
+    handler, calls = _scripted(error)
+
+    with pytest.raises(type(error)):
+        _client(handler).ask(_item({"visualizations": ["v1"]}))
+
+    assert len(calls) == 1
+    assert sleeps == []
+
+
+def test_transient_status_raises_with_body_once_retries_run_out(sleeps: list[float]) -> None:
+    handler, calls = _scripted(*[httpx.Response(502, text="Visualizations failed to execute.")] * 6)
+
+    with pytest.raises(httpx.HTTPStatusError, match="502.*failed to execute") as exc_info:
+        _client(handler).ask(_item({"visualizations": ["v1"]}))
+
+    assert exc_info.value.response.status_code == 502
+    assert len(calls) == 6
+
+
+@pytest.mark.parametrize("status", [400, 404, 422])
+def test_deterministic_client_error_is_not_retried(status: int, sleeps: list[float]) -> None:
+    handler, calls = _scripted(httpx.Response(status, text='{"reason": "NO_AVAILABLE_AGENT"}'))
+
+    with pytest.raises(httpx.HTTPStatusError, match="NO_AVAILABLE_AGENT"):
+        _client(handler).ask(_item({"visualizations": ["v1"]}))
+
+    assert len(calls) == 1
+    assert sleeps == []
+
+
+def test_default_timeout_is_two_minutes() -> None:
+    assert TakeawayClient(host="https://h", token="tok", workspace_id="ws")._client.timeout.read == 120.0
+
+
+def test_timeout_is_overridable() -> None:
+    assert TakeawayClient(host="https://h", token="tok", workspace_id="ws", timeout=30.0)._client.timeout.read == 30.0

From da07a7e9cf70d7cbdd7134e89fc4bb1d2922bb35 Mon Sep 17 00:00:00 2001
From: Roman Rakus <roman.rakus@gooddata.com>
Date: Thu, 22 Oct 2026 13:52:30 +0200
Subject: [PATCH 3/4] feat(gooddata-eval): add DocumentTakeawayEvaluator

The document_takeaway kind (LX-3167) has to prove first that a takeaway
is short and punchy: one finding for the page, not an executive
summary, whatever maxCharacters allows. Deterministic checks gate word
budget, headline length, sentence count, one paragraph, bold headline,
no headings, lists, ids or meta openers, and the length limit; an item's
format block switches a check off when its instruction asks for another
shape. word_count and headline_word_count go into the detail so they can
be published as trend scores.

Default judge criteria (one finding, no executive summary, chart
walk-through, background opener, figures, dates, causes outside the
data, chart description) are added unless the item opts out. Their
wording is calibrated against the judge with make calibrate-takeaway on
fixed good and bad texts; the reference good example used to fail the
cause criterion and an obvious executive summary used to pass.

noData decides the item alone, so an expected takeaway that came back
empty is a failure rather than an empty text to judge. The per-criterion
grading loop moves out of the summary evaluator into a shared helper
both evaluators use.

jira: LX-3167
risk: low
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 packages/gooddata-eval/AGENTS.md              |   4 +-
 packages/gooddata-eval/Makefile               |   4 +
 .../scripts/calibrate_takeaway_judge.py       |  75 ++++
 .../gooddata_eval/core/evaluators/__init__.py |   7 +-
 .../gooddata_eval/core/evaluators/_rubric.py  |  94 ++++
 .../gooddata_eval/core/evaluators/summary.py  |  76 +---
 .../gooddata_eval/core/evaluators/takeaway.py | 245 ++++++++++
 .../tests/fixtures/takeaway_calibration.json  |  68 +++
 packages/gooddata-eval/tests/test_runner.py   |   1 +
 .../tests/test_takeaway_evaluator.py          | 425 ++++++++++++++++++
 10 files changed, 932 insertions(+), 67 deletions(-)
 create mode 100644 packages/gooddata-eval/scripts/calibrate_takeaway_judge.py
 create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/evaluators/_rubric.py
 create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/evaluators/takeaway.py
 create mode 100644 packages/gooddata-eval/tests/fixtures/takeaway_calibration.json
 create mode 100644 packages/gooddata-eval/tests/test_takeaway_evaluator.py

diff --git a/packages/gooddata-eval/AGENTS.md b/packages/gooddata-eval/AGENTS.md
index 3c7846a5c..dbc663564 100644
--- a/packages/gooddata-eval/AGENTS.md
+++ b/packages/gooddata-eval/AGENTS.md
@@ -27,6 +27,7 @@ experiment. The newest and most actively developed package in the repo.
 | `core/agentic/` | multi-turn agentic evaluation per capability, **plus** all Langfuse trace polling and linking (`_langfuse.py`, `_trace_linker.py`) |
 | `core/chat/` | SSE client for the agent's streaming chat endpoint |
 | `core/summary/` | HTTP client for the dedicated dashboard-summary endpoint — a single-shot chat backend, not reporting |
+| `core/takeaway/` | HTTP client for the AI Publisher takeaway endpoint — a single-shot backend like `core/summary/`, keeping the whole response on `ChatResult.raw_response` |
 | `core/dataset/` | dataset format, loading, and `from_insights.py` — dataset generation from a workspace's real insights |
 | `core/evaluators/` | single-shot evaluators and their registry |
 | `core/langfuse/` | the whole Langfuse v4 client: `_env` (base URL + credentials), `otlp` (OTLP/JSON encoding), `experiment` (root-span construction, score targets), `item_scope` (an item's experiment scope sent to gen-ai as baggage, and which conversations joined), `observations` (trace reads), `client` (httpx calls), `sink` (single-shot results as experiments) |
@@ -59,7 +60,8 @@ implementation, sometimes under different `test_kind` strings (`search_tool` vs
 `expected_output: Any`. `expected_output` is deliberately untyped — each evaluator parses
 its own shape. `test_kind` on the item is what labels the result, not the evaluator class,
 which is why `knowledge_question` can reuse `GeneralQuestionEvaluator` verbatim.
-`dashboard_summary` items additionally need `summary_input`.
+`dashboard_summary` items additionally need `summary_input`, `document_takeaway` items
+`takeaway_input`.
 
 ## Running the pipeline
 
diff --git a/packages/gooddata-eval/Makefile b/packages/gooddata-eval/Makefile
index d1c4d1b3f..59c2f679e 100644
--- a/packages/gooddata-eval/Makefile
+++ b/packages/gooddata-eval/Makefile
@@ -1,2 +1,6 @@
 # (C) 2026 GoodData Corporation
 include ../../project_common.mk
+
+.PHONY: calibrate-takeaway
+calibrate-takeaway:
+	uv run --package gooddata-eval --extra llm-judge python scripts/calibrate_takeaway_judge.py
diff --git a/packages/gooddata-eval/scripts/calibrate_takeaway_judge.py b/packages/gooddata-eval/scripts/calibrate_takeaway_judge.py
new file mode 100644
index 000000000..ce7630650
--- /dev/null
+++ b/packages/gooddata-eval/scripts/calibrate_takeaway_judge.py
@@ -0,0 +1,75 @@
+# (C) 2026 GoodData Corporation
+"""Does the real judge put each calibration text on the right side of every default criterion?
+
+tests/test_takeaway_evaluator.py pins the wiring and the criterion wording; it cannot pin
+the verdict, because the verdict is an LLM's. This does, against the texts in
+tests/fixtures/takeaway_calibration.json:
+
+- a "pass" text must satisfy every default judge criterion on every run;
+- a "fail" text must violate each criterion it names on every run.
+
+Run it after changing any DEFAULT_MUST_INCLUDE / DEFAULT_MUST_NOT_INCLUDE wording:
+
+    OPENAI_API_KEY=... make calibrate-takeaway
+
+It is a script rather than a test because CI has no OPENAI_API_KEY. Exits 1 on any mismatch.
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+from gooddata_eval.core.evaluators.takeaway import (
+    DEFAULT_MUST_INCLUDE,
+    DEFAULT_MUST_NOT_INCLUDE,
+    DocumentTakeawayEvaluator,
+)
+from gooddata_eval.core.models import ChatResult, DatasetItem, TakeawayInput
+
+RUNS = 3
+FIXTURE = Path(__file__).resolve().parent.parent / "tests" / "fixtures" / "takeaway_calibration.json"
+
+CRITERIA = {f"default_include_{k}": k for k in DEFAULT_MUST_INCLUDE} | {
+    f"default_exclude_{k}": k for k in DEFAULT_MUST_NOT_INCLUDE
+}
+
+
+def main() -> int:
+    if not os.environ.get("OPENAI_API_KEY"):
+        raise SystemExit("OPENAI_API_KEY required")
+    fixture = json.loads(FIXTURE.read_text())
+    evaluator = DocumentTakeawayEvaluator()
+    print(f"judge model: {evaluator._positive_judge.model}, runs: {RUNS}")
+    mismatches = 0
+    for case in fixture["texts"]:
+        item = DatasetItem(
+            id=case["label"],
+            dataset_name="takeaway_calibration",
+            test_kind="document_takeaway",
+            question=fixture["question"],
+            expected_output={},
+            takeaway_input=TakeawayInput(visualizations=["calibration"]),
+        )
+        chat = ChatResult(text_response=case["text"], raw_response={"takeaway": case["text"], "noData": False})
+        satisfied = dict.fromkeys(CRITERIA.values(), 0)
+        for _ in range(RUNS):
+            detail = evaluator.evaluate(item, chat).detail
+            for key, name in CRITERIA.items():
+                satisfied[name] += detail.get(key) is True
+        want_pass = case["want"] == "pass"
+        must = dict.fromkeys(satisfied, RUNS) if want_pass else dict.fromkeys(case["criteria"], 0)
+        print(f"\n{case['label']} (want {case['want']})")
+        for name, count in satisfied.items():
+            flag = ""
+            if name in must:
+                ok = count == must[name]
+                mismatches += not ok
+                flag = "OK" if ok else "MISMATCH"
+            print(f"  {name:20} satisfied {count}/{RUNS}  {flag}")
+    print(f"\n{'all calibration texts on the right side' if not mismatches else f'{mismatches} mismatch(es)'}")
+    return 1 if mismatches else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py
index 30b154821..16a9e91b7 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py
@@ -20,7 +20,8 @@
     )
 }
 
-# LLM-judge evaluators (general_question, guardrail, dashboard_summary, knowledge_question)
+# LLM-judge evaluators (general_question, guardrail, dashboard_summary, knowledge_question,
+# document_takeaway)
 # require the [llm-judge] extra. Their modules are imported lazily on first use so the CLI
 # starts without openai.
 #
@@ -34,12 +35,14 @@
     "guardrail": "gooddata_eval.core.evaluators.guardrail",
     "dashboard_summary": "gooddata_eval.core.evaluators.summary",
     "knowledge_question": "gooddata_eval.core.evaluators.general_question",
+    "document_takeaway": "gooddata_eval.core.evaluators.takeaway",
 }
 _LAZY_EVALUATOR_CLASSES: dict[str, str] = {
     "general_question": "GeneralQuestionEvaluator",
     "guardrail": "GuardrailEvaluator",
     "dashboard_summary": "DashboardSummaryEvaluator",
     "knowledge_question": "GeneralQuestionEvaluator",
+    "document_takeaway": "DocumentTakeawayEvaluator",
 }
 
 
@@ -66,7 +69,7 @@ def supported_test_kinds() -> set[str]:
     """Return all supported test_kind values.
 
     LLM-judge kinds (general_question, guardrail, dashboard_summary,
-    knowledge_question) are excluded when the [llm-judge] extra (openai) is
+    knowledge_question, document_takeaway) are excluded when the [llm-judge] extra (openai) is
     not installed — those items are skipped rather than erroring out mid-run.
     """
     kinds = set(_EAGER_EVALUATORS)
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_rubric.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_rubric.py
new file mode 100644
index 000000000..26a6c3095
--- /dev/null
+++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_rubric.py
@@ -0,0 +1,94 @@
+# (C) 2026 GoodData Corporation
+"""Per-criterion LLM-judge grading shared by the rubric-scored single-shot evaluators."""
+
+from collections.abc import Sequence
+from typing import Any
+
+from gooddata_eval.core.evaluators._llm_judge import LLMJudge, score_run
+from gooddata_eval.core.evaluators.base import ItemEvaluation
+
+# A criterion keyed by the name its verdict is stored under in `detail`.
+Criterion = tuple[str, str]
+
+
+def grade_rubric(
+    positive_judge: LLMJudge,
+    violation_judge: LLMJudge,
+    *,
+    question: str,
+    actual: str,
+    must_include: Sequence[Criterion],
+    must_not_include: Sequence[Criterion],
+    rubric: Sequence[Criterion],
+    detail: dict[str, Any],
+    passed: bool = True,
+) -> ItemEvaluation:
+    """Grade every criterion with one judge request each and decide the item.
+
+    `must_include` and `must_not_include` gate the verdict, `rubric` only informs quality.
+    `must_not_include` criteria are put to `violation_judge` as a presence question and
+    inverted here. `detail` may already hold verdicts the caller decided without a judge,
+    with `passed` carrying their outcome; every bool in it counts towards quality.
+
+    A criterion the judge returns nothing readable for is *ungraded*: recorded without a
+    bool, so it neither fails the item nor enters the quality denominator, but it cannot
+    carry a pass either. When nothing that decides the item was graded, the run has no
+    verdict and comes back with `error` set.
+    """
+    # One judge request PER CRITERION, so an unreadable response has to be confined to
+    # its own criterion: letting it raise would discard every criterion already graded
+    # and abandon the ones after it, losing a 7-criterion item to one bad body.
+    decided_without_judge = passed
+    ungraded = 0
+    graded_gating = 0
+    graded_rubric = 0
+
+    def _grade(judge: LLMJudge, criterion: str, key: str, *, invert: bool = False) -> bool | None:
+        nonlocal ungraded
+        verdict = score_run(judge, input=question, expected_output=criterion, actual_output=actual)
+        if verdict.error is not None:
+            ungraded += 1
+            detail[f"{key}_reason"] = f"UNGRADED: {verdict.error}"
+            return None
+        # invert: the violation judge answers "is the characteristic present?", so the
+        # criterion is satisfied exactly when it says no.
+        ok = not verdict.passed if invert else verdict.passed
+        detail[key] = ok
+        detail[f"{key}_reason"] = verdict.reasoning
+        return ok
+
+    # `is True`, not `is not False`: a gating criterion the judge could not grade is not
+    # a failure, but it cannot carry a pass either.
+    for judge, criteria, invert in ((positive_judge, must_include, False), (violation_judge, must_not_include, True)):
+        for key, criterion in criteria:
+            ok = _grade(judge, criterion, key, invert=invert)
+            graded_gating += ok is not None
+            passed = passed and ok is True
+
+    for key, criterion in rubric:
+        graded_rubric += _grade(positive_judge, criterion, key) is not None
+
+    if ungraded:
+        detail["ungraded_criteria"] = ungraded
+    bool_checks = [v for v in detail.values() if isinstance(v, bool)]
+    quality = sum(1 for v in bool_checks if v) / len(bool_checks) if bool_checks else 0.0
+
+    # Keyed on the criteria that decide the verdict, not on any graded bool: rubric
+    # criteria never gate `passed`, so one graded rubric line must not certify a pass
+    # whose every mandatory fact went unassessed. A rubric-only item has no gating
+    # criteria, so there any graded line is a verdict. A caller-decided failure already
+    # is a verdict.
+    gating = len(must_include) + len(must_not_include)
+    judge_verdict = graded_gating if gating else graded_rubric
+    if ungraded and decided_without_judge and not judge_verdict:
+        return ItemEvaluation(
+            passed=False,
+            rank_key=(-1, quality),
+            detail=detail,
+            error=(
+                f"judge returned no readable verdict for any of the {gating or ungraded} criterion(s) "
+                "that decide this item."
+            ),
+        )
+
+    return ItemEvaluation(passed=passed, rank_key=(int(passed), quality), detail=detail)
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/summary.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/summary.py
index 0c680b0c8..46fb9e27e 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/summary.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/summary.py
@@ -27,7 +27,8 @@
 
 from typing import Any
 
-from gooddata_eval.core.evaluators._llm_judge import LLMJudge, score_run
+from gooddata_eval.core.evaluators._llm_judge import LLMJudge
+from gooddata_eval.core.evaluators._rubric import grade_rubric
 from gooddata_eval.core.evaluators._text_utils import extract_text
 from gooddata_eval.core.evaluators.base import ItemEvaluation
 from gooddata_eval.core.models import ChatResult, DatasetItem
@@ -74,66 +75,13 @@ def _criteria(expected_output: Any) -> tuple[list[str], list[str], list[str]]:
     def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation:
         actual = extract_text(chat_result)
         must_include, must_not_include, rubric = self._criteria(item.expected_output)
-
-        detail: dict[str, Any] = {"actual_output": actual}
-        passed = True
-
-        # One judge request PER CRITERION, so an unreadable response has to be confined to
-        # its own criterion: letting it raise would discard every criterion already graded
-        # and abandon the ones after it, losing a 7-criterion item to one bad body. An
-        # ungraded criterion is recorded but stored without a bool, which keeps it out of
-        # the failing list and the quality denominator below: counting it as failed would
-        # invent a score the judge never gave.
-        ungraded = 0
-
-        def _grade(judge: LLMJudge, criterion: str, key: str, *, invert: bool = False) -> bool | None:
-            nonlocal ungraded
-            verdict = score_run(judge, input=item.question, expected_output=criterion, actual_output=actual)
-            if verdict.error is not None:
-                ungraded += 1
-                detail[f"{key}_reason"] = f"UNGRADED: {verdict.error}"
-                return None
-            # invert: the violation judge answers "is the characteristic present?", so the
-            # criterion is satisfied exactly when it says no.
-            ok = not verdict.passed if invert else verdict.passed
-            detail[key] = ok
-            detail[f"{key}_reason"] = verdict.reasoning
-            return ok
-
-        # `is True`, not `is not False`: a gating criterion the judge could not grade is not
-        # a failure, but it cannot carry a pass either.
-        for i, criterion in enumerate(must_include):
-            ok = _grade(self._positive_judge, criterion, f"include_{i}")
-            passed = passed and ok is True
-
-        for i, criterion in enumerate(must_not_include):
-            ok = _grade(self._violation_judge, criterion, f"exclude_{i}", invert=True)
-            passed = passed and ok is True
-
-        for i, criterion in enumerate(rubric):
-            # Rubric criteria inform quality but never gate `passed`.
-            _grade(self._positive_judge, criterion, f"rubric_{i}")
-
-        if ungraded:
-            detail["ungraded_criteria"] = ungraded
-        bool_checks = [v for v in detail.values() if isinstance(v, bool)]
-        quality = sum(1 for v in bool_checks if v) / len(bool_checks) if bool_checks else 0.0
-
-        # Keyed on the criteria that decide the verdict, not on any graded bool: rubric
-        # criteria never gate `passed`, so one graded rubric line must not certify a pass
-        # whose every mandatory fact went unassessed. A rubric-only item has no gating
-        # criteria, so there any graded line is a verdict.
-        gating = len(must_include) + len(must_not_include)
-        graded_gating = [v for k, v in detail.items() if isinstance(v, bool) and not k.startswith("rubric_")]
-        if ungraded and not (graded_gating if gating else bool_checks):
-            return ItemEvaluation(
-                passed=False,
-                rank_key=(-1, quality),
-                detail=detail,
-                error=(
-                    f"judge returned no readable verdict for any of the {gating or ungraded} criterion(s) "
-                    "that decide this item."
-                ),
-            )
-
-        return ItemEvaluation(passed=passed, rank_key=(int(passed), quality), detail=detail)
+        return grade_rubric(
+            self._positive_judge,
+            self._violation_judge,
+            question=item.question,
+            actual=actual,
+            must_include=[(f"include_{i}", c) for i, c in enumerate(must_include)],
+            must_not_include=[(f"exclude_{i}", c) for i, c in enumerate(must_not_include)],
+            rubric=[(f"rubric_{i}", c) for i, c in enumerate(rubric)],
+            detail={"actual_output": actual},
+        )
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/takeaway.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/takeaway.py
new file mode 100644
index 000000000..494910938
--- /dev/null
+++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/takeaway.py
@@ -0,0 +1,245 @@
+# (C) 2026 GoodData Corporation
+"""Evaluator for document_takeaway: deterministic format checks plus an LLM-judge rubric.
+
+A takeaway is one finding for a page, readable in about five seconds: a bold headline,
+at most two short sentences, one paragraph, ≤ 50 words whatever ``maxCharacters`` allows.
+`expected_output` is:
+
+    {
+      "no_data":          false,
+      "format":           {...overrides of _DEFAULT_FORMAT...},
+      "default_rules":    true,
+      "must_include":     ["...criteria the takeaway must satisfy..."],
+      "must_not_include": ["...characteristics the takeaway must not show..."],
+      "rubric":           ["...soft quality dimensions..."]
+    }
+
+Every key is optional. The format checks need no judge and gate the verdict. A ``format``
+value of ``false`` or ``null`` switches its check off, for an item whose instruction asks
+for another shape (a list, no headline). Unless ``default_rules`` is false, the
+DEFAULT_MUST_INCLUDE / DEFAULT_MUST_NOT_INCLUDE judge criteria are added to the item's
+own, under ``default_`` keys. Judge criteria grade like ``dashboard_summary``.
+
+``no_data`` decides the item alone: the endpoint's ``noData`` must equal it, and a
+``noData`` answer carries no takeaway, so nothing else is graded. An expected takeaway
+that came back as ``noData`` is a failure, not an empty text to judge.
+"""
+
+import re
+from typing import Any
+
+from gooddata_eval.core.evaluators._llm_judge import LLMJudge
+from gooddata_eval.core.evaluators._rubric import Criterion, grade_rubric
+from gooddata_eval.core.evaluators.base import ItemEvaluation
+from gooddata_eval.core.models import ChatResult, DatasetItem
+
+# The judge criteria below are calibrated against tests/fixtures/takeaway_calibration.json
+# with `make calibrate-takeaway`; re-run it after changing any of their wording.
+DEFAULT_MUST_INCLUDE: dict[str, str] = {
+    "one_finding": "A reader gets one clear main finding from the first line alone.",
+}
+
+DEFAULT_MUST_NOT_INCLUDE: dict[str, str] = {
+    "executive_summary": (
+        "Reads as an executive summary or report: it contains section labels such as 'Summary', "
+        "'Key takeaways' or 'Things to watch', or lists several separate findings one after another, "
+        "or gives recommendations, an outlook or a closing conclusion."
+    ),
+    "chart_walkthrough": "Walks through the charts one by one instead of stating one finding that connects them.",
+    "background_opener": "Opens with background or a description of the page instead of the finding itself.",
+    "exact_figure": (
+        "States a figure in digits: an amount, count, percentage or share, including rounded or hedged ones "
+        "such as 'about $270M', 'roughly 1.2K', 'in the low $100Ks' or '~40%'. A rough size or direction in words, such as "
+        "'roughly doubled', 'about a third' or 'most orders', does not count. Numbers that are part of a "
+        "visualization or metric title, such as 'Top 10', do not count."
+    ),
+    "date_or_period": (
+        "Names a specific date, month, quarter, year or named period. Relative references such as "
+        "'this period' do not count."
+    ),
+    "unshown_cause": (
+        "Makes an explicit causal claim (such as 'because', 'due to' or 'driven by') that attributes a change "
+        "to a factor that is not in the data, such as a campaign, a season or a market event. Comparisons "
+        "between segments, and naming the segment in the data that led a change (such as 'led by the "
+        "Nordics'), are not causal claims."
+    ),
+    "chart_description": "Describes what a chart displays instead of what the data means.",
+}
+
+_DEFAULT_FORMAT: dict[str, Any] = {
+    "bold_headline": True,
+    "max_headline_words": 10,
+    "max_sentences": 2,
+    "max_words": 50,
+    "single_paragraph": True,
+    "no_lists": True,
+    "no_headings": True,
+}
+
+_POSITIVE_STEPS = [
+    "Read the INPUT (a label for the page the takeaway was written for) and the EXPECTED OUTPUT "
+    "(one criterion the takeaway must satisfy).",
+    "Read the ACTUAL OUTPUT (the generated takeaway).",
+    "Score 1 if the actual output clearly satisfies the criterion.",
+    "Score 0 if the criterion is missing, contradicted, or only partially addressed.",
+]
+
+# A presence question, inverted in code -- see the same split in summary.py.
+_VIOLATION_STEPS = [
+    "Read the CHARACTERISTIC described in EXPECTED OUTPUT, including any exceptions it states.",
+    "Read the ACTUAL OUTPUT (the generated takeaway).",
+    "Score 1 if the actual output clearly exhibits the described characteristic.",
+    "Score 0 if it does not exhibit it.",
+]
+
+_HEADLINE = re.compile(r"^\s*\*\*(.+?)\*\*", re.DOTALL)
+# A terminator, any closing quotes or brackets, then whitespace before the next sentence.
+_SENTENCE_BREAK = re.compile(r"[.!?]+[\"'”’»)\]]*\s+(?=\S)")
+_SENTENCE_OPENERS = "\"'“‘„«(["
+_ABBREVIATIONS = frozenset({"vs", "e.g", "i.e", "u.s", "cca", "např", "tj", "tzn", "apod", "resp", "atd"})
+_BLANK_LINE = re.compile(r"\n\s*\n")
+_HEADING_LINE = re.compile(r"^\s*#{1,6}\s", re.MULTILINE)
+_LIST_LINE = re.compile(r"^\s*(?:[-*•]|\d+[.)])\s", re.MULTILINE)
+_IDS = re.compile(
+    r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|\{\w+/[^}]*\}|\blabel\.\w",
+    re.IGNORECASE,
+)
+_META_OPENER = re.compile(
+    r"^(?:summary|key takeaways?|based on the data|visualization metadata|this chart shows)\b",
+    re.IGNORECASE,
+)
+
+
+def _words(text: str) -> int:
+    return sum(1 for token in text.replace("**", " ").split() if any(ch.isalnum() for ch in token))
+
+
+def _limit_on(value: Any) -> bool:
+    return value is not None and value is not False
+
+
+def _sentences(text: str) -> int:
+    """Sentences in `text`; a break needs the next sentence to open with a capital, digit or quote.
+
+    That keeps "vs. last", "U.S. sales" or "např. v" from splitting one sentence in two; a
+    terminator right after a known abbreviation ("vs. Europe", "e.g. Nordics") is no break.
+    """
+    plain = text.replace("**", "").strip()
+    if not plain:
+        return 0
+    breaks = 0
+    for match in _SENTENCE_BREAK.finditer(plain):
+        nxt = plain[match.end()]
+        token = (plain[: match.start()].rsplit(maxsplit=1) or [""])[-1].lstrip("\"'“‘„«([").lower()
+        if token in _ABBREVIATIONS:
+            continue
+        breaks += nxt.isupper() or nxt.isdigit() or nxt in _SENTENCE_OPENERS
+    return breaks + 1
+
+
+def _has_meta_opener(text: str, headline: str | None, body: str) -> bool:
+    openers = [line.strip().lstrip("#*").strip() for line in text.splitlines()]
+    openers += [headline or "", body.strip()]
+    return any(_META_OPENER.match(opener) for opener in openers)
+
+
+class DocumentTakeawayEvaluator:
+    test_kind = "document_takeaway"
+
+    def __init__(self):
+        self._positive_judge = LLMJudge(evaluation_steps=_POSITIVE_STEPS)
+        self._violation_judge = LLMJudge(evaluation_steps=_VIOLATION_STEPS)
+
+    @staticmethod
+    def validate(item: DatasetItem) -> None:
+        """Reject an item whose `expected_output` this evaluator cannot read, before any request.
+
+        Raises:
+            ValueError: `format` is not an object, carries a key the checks do not know, or a criteria key
+                is not a JSON array.
+        """
+        expected = item.expected_output if isinstance(item.expected_output, dict) else {}
+        overrides = expected.get("format") or {}
+        if not isinstance(overrides, dict):
+            raise ValueError(f"document_takeaway item '{item.id}': format must be a JSON object")
+        unknown = sorted(set(overrides) - set(_DEFAULT_FORMAT))
+        if unknown:
+            raise ValueError(
+                f"document_takeaway item '{item.id}': unknown format key(s) {unknown}; known: {sorted(_DEFAULT_FORMAT)}"
+            )
+        for key in ("must_include", "must_not_include", "rubric"):
+            if not isinstance(expected.get(key, []), list):
+                raise ValueError(f"document_takeaway item '{item.id}': {key} must be a JSON array")
+
+    def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation:
+        self.validate(item)
+        expected: dict[str, Any] = item.expected_output if isinstance(item.expected_output, dict) else {}
+        raw = chat_result.raw_response or {}
+        returned_no_data = raw.get("noData") is True
+        text = chat_result.text_response or ""
+
+        expected_no_data = expected.get("no_data") is True
+        if expected_no_data or returned_no_data:
+            ok = expected_no_data and returned_no_data and raw.get("takeaway") is None
+            detail: dict[str, Any] = {"actual_output": text, "no_data": ok}
+            return ItemEvaluation(passed=ok, rank_key=(int(ok), float(ok)), detail=detail)
+
+        detail = {"actual_output": text}
+        detail.update(self._format_checks(item, expected, text))
+        passed = all(v for v in detail.values() if isinstance(v, bool))
+
+        must_include: list[Criterion] = []
+        must_not_include: list[Criterion] = []
+        if expected.get("default_rules", True) is not False:
+            must_include += [(f"default_include_{k}", c) for k, c in DEFAULT_MUST_INCLUDE.items()]
+            must_not_include += [(f"default_exclude_{k}", c) for k, c in DEFAULT_MUST_NOT_INCLUDE.items()]
+        must_include += [(f"include_{i}", str(c)) for i, c in enumerate(expected.get("must_include", []))]
+        must_not_include += [(f"exclude_{i}", str(c)) for i, c in enumerate(expected.get("must_not_include", []))]
+        rubric = [(f"rubric_{i}", str(c)) for i, c in enumerate(expected.get("rubric", []))]
+
+        return grade_rubric(
+            self._positive_judge,
+            self._violation_judge,
+            question=item.question,
+            actual=text,
+            must_include=must_include,
+            must_not_include=must_not_include,
+            rubric=rubric,
+            detail=detail,
+            passed=passed,
+        )
+
+    @staticmethod
+    def _format_checks(item: DatasetItem, expected: dict[str, Any], text: str) -> dict[str, Any]:
+        """Deterministic checks keyed by name, plus `word_count` and `headline_word_count`."""
+        fmt = {**_DEFAULT_FORMAT, **(expected.get("format") or {})}
+        checks: dict[str, Any] = {}
+
+        match = _HEADLINE.match(text) if fmt["bold_headline"] else None
+        headline = match.group(1).strip() if match else None
+        body = text[match.end() :] if match else text
+        word_count = _words(text)
+        headline_word_count = _words(headline) if headline else 0
+        checks["word_count"] = word_count
+        checks["headline_word_count"] = headline_word_count
+
+        if _limit_on(fmt["max_words"]):
+            checks["word_budget"] = word_count <= fmt["max_words"]
+        if fmt["bold_headline"]:
+            checks["bold_headline"] = headline is not None
+            if _limit_on(fmt["max_headline_words"]):
+                checks["headline_words"] = headline_word_count <= fmt["max_headline_words"]
+        if _limit_on(fmt["max_sentences"]):
+            checks["max_sentences"] = _sentences(body) <= fmt["max_sentences"]
+        if fmt["single_paragraph"]:
+            checks["single_paragraph"] = not _BLANK_LINE.search(text.strip()) and text.strip().count("\n") <= 1
+        max_characters = item.takeaway_input.max_characters if item.takeaway_input else None
+        if max_characters is not None:
+            checks["length"] = len(text) <= max_characters
+        if fmt["no_headings"]:
+            checks["no_headings"] = not _HEADING_LINE.search(text)
+        if fmt["no_lists"]:
+            checks["no_lists"] = not _LIST_LINE.search(text)
+        checks["no_ids"] = not _IDS.search(text)
+        checks["no_meta"] = not _has_meta_opener(text, headline, body)
+        return checks
diff --git a/packages/gooddata-eval/tests/fixtures/takeaway_calibration.json b/packages/gooddata-eval/tests/fixtures/takeaway_calibration.json
new file mode 100644
index 000000000..6ef3502de
--- /dev/null
+++ b/packages/gooddata-eval/tests/fixtures/takeaway_calibration.json
@@ -0,0 +1,68 @@
+{
+  "question": "Takeaway for one dashboard page",
+  "texts": [
+    {
+      "label": "new markets (reference good example)",
+      "want": "pass",
+      "text": "**Growth now comes from new markets.** Sales rose again this period, led by the Nordics, while core markets held flat."
+    },
+    {
+      "label": "returning customers",
+      "want": "pass",
+      "text": "**Returning customers are the gap.** Almost all active customers are new, and the biggest spenders carry a large share of sales."
+    },
+    {
+      "label": "bigger baskets",
+      "want": "pass",
+      "text": "**Fewer orders, bigger baskets.** Order count dipped while the average order grew, keeping total sales steady."
+    },
+    {
+      "label": "category concentration",
+      "want": "pass",
+      "text": "**Electronics carries the product mix.** It leads every other category in sales, and the gap to the rest keeps widening."
+    },
+    {
+      "label": "rough size in words (prompt phrasing)",
+      "want": "pass",
+      "text": "**Audio and video now carry the store.** They bring in about a third of sales, and their sales roughly doubled compared with last period."
+    },
+    {
+      "label": "executive summary",
+      "want": "fail",
+      "criteria": ["executive_summary", "exact_figure", "date_or_period"],
+      "text": "**Summary:** Net Sales were $4.2M in Q3 2026, up 12%. Key takeaways: Electronics leads with 38% share; returns fell to 2.1%. Things to watch: inventory in Furniture is rising, and discounts may erode margin next quarter."
+    },
+    {
+      "label": "chart walk-through",
+      "want": "fail",
+      "criteria": ["chart_walkthrough", "exact_figure"],
+      "text": "**Sales overview.** Total Sales Trend shows sales in the low $100Ks, Net Sales by Product Category shows Audio & Video leading, and Order Breakdown shows most orders are Processed."
+    },
+    {
+      "label": "hedged figures (real output)",
+      "want": "fail",
+      "criteria": ["exact_figure"],
+      "text": "**Solid top-line scale** Total Sales are about $270M this period, supported by a customer base of roughly 1.2K Total Customers."
+    },
+    {
+      "label": "percentage with a tilde (real output)",
+      "want": "fail",
+      "criteria": [
+        "exact_figure"
+      ],
+      "text": "**Sales mix:** Audio & Video Accessories drive ~40% of Net Sales, and most orders are Processed."
+    },
+    {
+      "label": "page description opener",
+      "want": "fail",
+      "criteria": ["background_opener"],
+      "text": "This page shows sales, orders and customers for the online store. Sales rose while orders held flat."
+    },
+    {
+      "label": "external cause",
+      "want": "fail",
+      "criteria": ["unshown_cause"],
+      "text": "**Sales jumped in the West.** The rise came because the summer campaign drew in new buyers."
+    }
+  ]
+}
diff --git a/packages/gooddata-eval/tests/test_runner.py b/packages/gooddata-eval/tests/test_runner.py
index eb7a2bad2..6094621b8 100644
--- a/packages/gooddata-eval/tests/test_runner.py
+++ b/packages/gooddata-eval/tests/test_runner.py
@@ -134,6 +134,7 @@ def test_run_items_routes_all_supported_kinds():
         "guardrail",
         "dashboard_summary",
         "knowledge_question",
+        "document_takeaway",
     }
     assert expected_kinds == supported_test_kinds()
 
diff --git a/packages/gooddata-eval/tests/test_takeaway_evaluator.py b/packages/gooddata-eval/tests/test_takeaway_evaluator.py
new file mode 100644
index 000000000..26a93d780
--- /dev/null
+++ b/packages/gooddata-eval/tests/test_takeaway_evaluator.py
@@ -0,0 +1,425 @@
+# (C) 2026 GoodData Corporation
+from typing import Any
+from unittest.mock import MagicMock, patch
+
+import pytest
+from gooddata_eval.core.evaluators import get_evaluator, supported_test_kinds
+from gooddata_eval.core.evaluators.takeaway import (
+    DEFAULT_MUST_INCLUDE,
+    DEFAULT_MUST_NOT_INCLUDE,
+    DocumentTakeawayEvaluator,
+    _sentences,
+)
+from gooddata_eval.core.models import ChatResult, DatasetItem, TakeawayInput
+
+_GOOD = "**Growth now comes from new markets.** Sales rose again this period, led by the Nordics, while core markets held flat."
+
+
+def _make_evaluator(*, judge_passes: bool = True) -> DocumentTakeawayEvaluator:
+    """An evaluator whose judges grade every criterion as satisfied (or every one as not)."""
+    with patch("openai.OpenAI"), patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}):
+        ev = DocumentTakeawayEvaluator()
+    ev._positive_judge.score = MagicMock(return_value=(judge_passes, "positive"))
+    ev._violation_judge.score = MagicMock(return_value=(not judge_passes, "violation"))
+    return ev
+
+
+def _item(expected_output: Any = None, *, max_characters: int | None = None) -> DatasetItem:
+    return DatasetItem(
+        id="t1",
+        dataset_name="d",
+        test_kind="document_takeaway",
+        question="P01 default format, Sales page",
+        expected_output=expected_output if expected_output is not None else {},
+        takeaway_input=TakeawayInput(visualizations=["v1"], max_characters=max_characters),
+    )
+
+
+def _chat(text: str | None = _GOOD, *, no_data: bool = False) -> ChatResult:
+    raw = {"takeaway": text, "noData": no_data, "agentId": "a", "contextOmitted": []}
+    return ChatResult(text_response=text or "", raw_response=raw)
+
+
+def _judge_calls(ev: DocumentTakeawayEvaluator) -> int:
+    return ev._positive_judge.score.call_count + ev._violation_judge.score.call_count
+
+
+def test_good_takeaway_passes_every_check() -> None:
+    res = _make_evaluator().evaluate(_item(), _chat())
+
+    assert res.passed is True
+    assert res.error is None
+    for key in (
+        "word_budget",
+        "headline_words",
+        "max_sentences",
+        "single_paragraph",
+        "bold_headline",
+        "no_headings",
+        "no_lists",
+        "no_ids",
+        "no_meta",
+    ):
+        assert res.detail[key] is True, key
+
+
+def test_word_count_scores_reported() -> None:
+    res = _make_evaluator().evaluate(_item(), _chat())
+
+    assert res.detail["word_count"] == 20
+    assert res.detail["headline_word_count"] == 6
+
+
+def test_word_budget_ignores_large_max_characters() -> None:
+    text = "**Sales grew.** " + " ".join(["word"] * 53) + "."
+
+    res = _make_evaluator().evaluate(_item(max_characters=1000), _chat(text))
+
+    assert res.detail["word_count"] == 55
+    assert res.detail["word_budget"] is False
+    assert res.detail["length"] is True
+    assert res.passed is False
+
+
+def test_headline_over_ten_words_fails() -> None:
+    text = "**Sales in every region grew faster than any other metric this time.** Margins held."
+
+    res = _make_evaluator().evaluate(_item(), _chat(text))
+
+    assert res.detail["headline_word_count"] == 12
+    assert res.detail["headline_words"] is False
+    assert res.passed is False
+
+
+@pytest.mark.parametrize(
+    ("body", "count"),
+    [
+        ("Sales rose 3% vs. last period. Nordics led.", 2),
+        ("U.S. sales grew. Europe held flat.", 2),
+        ("Prodeje rostly, např. v Praze. Brno stagnovalo.", 2),
+        ("Prodeje rostly. Živé kanály stagnovaly.", 2),
+        ("Sales rose **sharply.** Orders fell. Returns grew.", 3),
+        ('Sales are "flat." Orders rose.', 2),
+        ("Margins held at 3.5% overall.", 1),
+        ("Sales grew. 2 regions lead.", 2),
+        ("Sales grew", 1),
+        ("Online outgrew retail vs. Europe in 2024.", 1),
+        ("Sales stay below vs. 2024 levels.", 1),
+        ("Growth came from new markets, e.g. Nordics. Core held.", 2),
+        ("Tržby rostly, tj. Praha vedla. Brno stagnovalo.", 2),
+        ("Sales grew (cca. 5 regions). Core held.", 2),
+        ("... Sales grew.", 2),
+        ("", 0),
+        ("  ** **  ", 0),
+    ],
+)
+def test_sentence_count(body: str, count: int) -> None:
+    assert _sentences(body) == count
+
+
+def test_bold_inside_the_body_still_counts_as_a_break() -> None:
+    res = _make_evaluator().evaluate(
+        _item(), _chat("**Sales grew.** Sales rose **sharply.** Orders fell. Returns grew.")
+    )
+
+    assert res.detail["max_sentences"] is False
+
+
+@pytest.mark.parametrize(
+    ("text", "ok"),
+    [
+        ("**Sales grew.** Margins held. Costs fell.", True),
+        ("**Sales grew.** Margins held. Costs fell. Returns rose.", False),
+        ("**Total is $4.2M.** Margins held at 3.5% overall.", True),
+    ],
+)
+def test_max_sentences_counts_after_the_headline(text: str, ok: bool) -> None:
+    res = _make_evaluator().evaluate(_item(), _chat(text))
+
+    assert res.detail["max_sentences"] is ok
+
+
+def test_max_sentences_counts_whole_text_without_headline() -> None:
+    res = _make_evaluator().evaluate(
+        _item({"format": {"bold_headline": False, "max_sentences": 1}}), _chat("Sales grew. Margins held.")
+    )
+
+    assert res.detail["max_sentences"] is False
+    assert "bold_headline" not in res.detail
+    assert "headline_words" not in res.detail
+    assert res.detail["headline_word_count"] == 0
+
+
+@pytest.mark.parametrize(
+    ("text", "ok"),
+    [
+        (_GOOD, True),
+        ("**Sales grew.**\nMargins held.", True),
+        ("**Sales grew.**\n\nMargins held.", False),
+        ("**Sales grew.**\nMargins held.\nCosts fell.", False),
+    ],
+)
+def test_single_paragraph(text: str, ok: bool) -> None:
+    res = _make_evaluator().evaluate(_item(), _chat(text))
+
+    assert res.detail["single_paragraph"] is ok
+
+
+@pytest.mark.parametrize(("limit", "ok"), [(len(_GOOD), True), (len(_GOOD) - 1, False)])
+def test_length_within_max_characters(limit: int, ok: bool) -> None:
+    res = _make_evaluator().evaluate(_item(max_characters=limit), _chat())
+
+    assert res.detail["length"] is ok
+
+
+def test_length_not_checked_without_max_characters() -> None:
+    res = _make_evaluator().evaluate(_item(), _chat())
+
+    assert "length" not in res.detail
+
+
+@pytest.mark.parametrize(
+    ("text", "ok"),
+    [
+        (_GOOD, True),
+        ("  **Sales grew.** Margins held.", True),
+        ("Sales grew. **Margins held.**", False),
+        ("Sales grew and margins held.", False),
+    ],
+)
+def test_bold_headline(text: str, ok: bool) -> None:
+    res = _make_evaluator().evaluate(_item(), _chat(text))
+
+    assert res.detail["bold_headline"] is ok
+
+
+@pytest.mark.parametrize(
+    ("text", "ok"),
+    [
+        (_GOOD, True),
+        ("## Sales grew\nMargins held.", False),
+        ("# Sales grew\nMargins held.", False),
+        ("#1 seller holds the lead.", True),
+        ("**Sales grew.**\n#2 brand closes the gap.", True),
+    ],
+)
+def test_no_headings(text: str, ok: bool) -> None:
+    res = _make_evaluator().evaluate(_item(), _chat(text))
+
+    assert res.detail["no_headings"] is ok
+
+
+@pytest.mark.parametrize(
+    ("text", "ok"),
+    [
+        (_GOOD, True),
+        ("**Sales grew.**\n- Margins held.", False),
+        ("**Sales grew.**\n• Margins held.", False),
+        ("**Sales grew.**\n1. Margins held.", False),
+        ("**Sales grew.**\n2) Margins held.", False),
+    ],
+)
+def test_no_lists(text: str, ok: bool) -> None:
+    res = _make_evaluator().evaluate(_item(), _chat(text))
+
+    assert res.detail["no_lists"] is ok
+
+
+def test_format_override_allows_lists() -> None:
+    text = "- Sales grew.\n- Margins held.\n- Costs fell."
+    fmt = {"no_lists": False, "bold_headline": False, "max_sentences": None, "single_paragraph": False}
+
+    res = _make_evaluator().evaluate(_item({"format": fmt}), _chat(text))
+
+    assert "no_lists" not in res.detail
+    assert "max_sentences" not in res.detail
+    assert "single_paragraph" not in res.detail
+    assert res.detail["word_budget"] is True
+    assert res.passed is True
+
+
+@pytest.mark.parametrize(
+    ("limit", "check"),
+    [("max_words", "word_budget"), ("max_headline_words", "headline_words"), ("max_sentences", "max_sentences")],
+)
+@pytest.mark.parametrize("off", [None, False])
+def test_format_limit_switched_off(limit: str, check: str, off: bool | None) -> None:
+    res = _make_evaluator().evaluate(_item({"format": {limit: off}}), _chat())
+
+    assert check not in res.detail
+    assert res.passed is True
+
+
+@pytest.mark.parametrize(
+    ("text", "ok"),
+    [
+        (_GOOD, True),
+        ("**Sales grew.** Driven by 25bdccd8-ef39-40a5-a224-6031d0ab470c.", False),
+        ("**Sales grew.** Driven by {metric/revenue}.", False),
+        ("**Sales grew.** Led by label.region.name values.", False),
+        ("**Sales grew.** The label. Margins held.", True),
+    ],
+)
+def test_no_ids(text: str, ok: bool) -> None:
+    res = _make_evaluator().evaluate(_item(), _chat(text))
+
+    assert res.detail["no_ids"] is ok
+
+
+@pytest.mark.parametrize(
+    ("text", "ok"),
+    [
+        (_GOOD, True),
+        ("**Summary:** Sales grew.", False),
+        ("**Key takeaways** Sales grew.", False),
+        ("**Sales grew.** Based on the data, margins held.", False),
+        ("This chart shows sales growing.", False),
+        ("**Sales grew.**\nVisualization metadata says margins held.", False),
+        ("**Sales grew.** The quarterly summary confirms it.", True),
+    ],
+)
+def test_no_meta(text: str, ok: bool) -> None:
+    res = _make_evaluator().evaluate(_item(), _chat(text))
+
+    assert res.detail["no_meta"] is ok
+
+
+def test_no_data_expected_and_returned_passes_without_judge() -> None:
+    ev = _make_evaluator()
+
+    res = ev.evaluate(_item({"no_data": True}), _chat(None, no_data=True))
+
+    assert res.passed is True
+    assert res.detail["no_data"] is True
+    assert _judge_calls(ev) == 0
+    assert "word_budget" not in res.detail
+
+
+@pytest.mark.parametrize(
+    ("expected", "text", "no_data"),
+    [(False, None, True), (True, _GOOD, False), (True, _GOOD, True)],
+)
+def test_no_data_mismatch_fails(expected: bool, text: str | None, no_data: bool) -> None:
+    ev = _make_evaluator()
+
+    res = ev.evaluate(_item({"no_data": expected}), _chat(text, no_data=no_data))
+
+    assert res.passed is False
+    assert res.detail["no_data"] is False
+    assert _judge_calls(ev) == 0
+
+
+def test_deterministic_failure_still_grades_the_judge_criteria() -> None:
+    ev = _make_evaluator()
+
+    res = ev.evaluate(_item(), _chat("Sales grew and margins held."))
+
+    assert res.passed is False
+    assert _judge_calls(ev) == len(DEFAULT_MUST_INCLUDE) + len(DEFAULT_MUST_NOT_INCLUDE)
+
+
+def test_judge_violation_fails_the_item() -> None:
+    res = _make_evaluator(judge_passes=False).evaluate(_item(), _chat())
+
+    assert res.passed is False
+    assert res.detail["default_exclude_exact_figure"] is False
+    assert res.detail["default_include_one_finding"] is False
+
+
+def test_default_rules_added_unless_disabled() -> None:
+    ev = _make_evaluator()
+    res = ev.evaluate(_item({"must_include": ["is about returning customers"]}), _chat())
+
+    assert ev._positive_judge.score.call_count == len(DEFAULT_MUST_INCLUDE) + 1
+    assert ev._violation_judge.score.call_count == len(DEFAULT_MUST_NOT_INCLUDE)
+    assert res.detail["include_0"] is True
+    assert all(f"default_exclude_{key}" in res.detail for key in DEFAULT_MUST_NOT_INCLUDE)
+
+    ev = _make_evaluator()
+    res = ev.evaluate(_item({"default_rules": False, "must_not_include": ["mentions a campaign"]}), _chat())
+
+    assert _judge_calls(ev) == 1
+    assert res.detail["exclude_0"] is True
+    assert not any(key.startswith("default_") for key in res.detail)
+
+
+def test_item_criteria_are_judged_against_the_takeaway() -> None:
+    ev = _make_evaluator()
+    ev.evaluate(_item({"default_rules": False, "rubric": ["plain language"]}), _chat())
+
+    ev._positive_judge.score.assert_called_once_with(
+        input="P01 default format, Sales page", expected_output="plain language", actual_output=_GOOD
+    )
+
+
+def test_title_number_criterion_text_present() -> None:
+    criterion = DEFAULT_MUST_NOT_INCLUDE["exact_figure"]
+
+    assert "Top 10" in criterion
+    assert "in digits" in criterion
+    assert "'about $270M'" in criterion
+    assert "'about a third'" in criterion
+    assert "'roughly doubled'" in criterion
+
+
+def test_calibrated_criterion_wording() -> None:
+    assert set(DEFAULT_MUST_NOT_INCLUDE) == {
+        "executive_summary",
+        "chart_walkthrough",
+        "background_opener",
+        "exact_figure",
+        "date_or_period",
+        "unshown_cause",
+        "chart_description",
+    }
+    assert "Things to watch" in DEFAULT_MUST_NOT_INCLUDE["executive_summary"]
+    assert "led by" in DEFAULT_MUST_NOT_INCLUDE["unshown_cause"]
+    assert set(DEFAULT_MUST_INCLUDE) == {"one_finding"}
+
+
+def test_registered_as_lazy_llm_judge_kind() -> None:
+    assert "document_takeaway" in supported_test_kinds()
+    with patch("openai.OpenAI"), patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}):
+        assert isinstance(get_evaluator("document_takeaway"), DocumentTakeawayEvaluator)
+
+
+def test_no_data_not_stored_as_a_bool_when_a_takeaway_is_expected() -> None:
+    res = _make_evaluator().evaluate(_item(), _chat())
+
+    assert not isinstance(res.detail.get("no_data"), bool)
+
+
+def test_unknown_format_key_is_rejected() -> None:
+    with pytest.raises(ValueError, match="max_sentence"):
+        _make_evaluator().evaluate(_item({"format": {"max_sentence": 1}}), _chat())
+
+
+def test_validate_rejects_an_unknown_format_key_without_a_judge_call() -> None:
+    ev = _make_evaluator()
+
+    with pytest.raises(ValueError, match="'t1'.*max_sentence"):
+        ev.validate(_item({"format": {"max_sentence": 1}}))
+
+    assert _judge_calls(ev) == 0
+
+
+def test_validate_rejects_a_format_that_is_not_an_object() -> None:
+    with pytest.raises(ValueError, match="'t1'.*format must be a JSON object"):
+        _make_evaluator().validate(_item({"format": ["no_lists"]}))
+
+
+@pytest.mark.parametrize("key", ["must_include", "must_not_include", "rubric"])
+@pytest.mark.parametrize("value", [None, "one criterion", {"a": "b"}])
+def test_validate_rejects_criteria_that_are_not_a_list(key: str, value: Any) -> None:
+    with pytest.raises(ValueError, match=f"'t1'.*{key} must be a JSON array"):
+        _make_evaluator().validate(_item({key: value}))
+
+
+def test_validate_accepts_absent_and_empty_criteria() -> None:
+    ev = _make_evaluator()
+    ev.validate(_item({}))
+    ev.validate(_item({"must_include": [], "must_not_include": [], "rubric": []}))
+
+
+def test_validate_accepts_a_valid_item() -> None:
+    _make_evaluator().validate(_item({"format": {"no_lists": False}, "must_include": ["x"]}))

From 8bd4fd63931bce46a8949658247f11d37b2c5c61 Mon Sep 17 00:00:00 2001
From: Roman Rakus <roman.rakus@gooddata.com>
Date: Thu, 22 Oct 2026 14:16:35 +0200
Subject: [PATCH 4/4] feat(gooddata-eval): route takeaway items and add the CI
 entry point

LX-3167 runs document_takeaway in the scheduled LLM eval CI through a
Tavern shim, and by hand through the CLI.

The CLI's routing backend becomes a test_kind -> client map, so
document_takeaway items go to the takeaway endpoint next to
dashboard_summary, and every client is closed. Items pulled with
--langfuse-dataset carry takeaway_input in metadata with a plain label
as input, which the loader already reads.

evaluate_document_takeaway runs an item K times, applies the gate and
writes the Langfuse experiment item through the same run context, gate
scores and metadata as the agentic kinds, so the nightly report and PR
comment read it unchanged. Every check is scored per run, with
word_count and headline_word_count as numeric trend scores. HTTP errors
and a missing takeaway_input surface as item errors, never as judged
failures. The README documents the kind and its item shape.

jira: LX-3167
risk: low
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 packages/gooddata-eval/AGENTS.md              |   2 +-
 packages/gooddata-eval/README.md              |  68 +++-
 .../src/gooddata_eval/cli/main.py             |  28 +-
 .../gooddata_eval/core/takeaway/evaluate.py   | 275 ++++++++++++++
 packages/gooddata-eval/tests/test_cli.py      |  51 +++
 .../tests/test_takeaway_evaluate.py           | 349 ++++++++++++++++++
 6 files changed, 757 insertions(+), 16 deletions(-)
 create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/takeaway/evaluate.py
 create mode 100644 packages/gooddata-eval/tests/test_takeaway_evaluate.py

diff --git a/packages/gooddata-eval/AGENTS.md b/packages/gooddata-eval/AGENTS.md
index dbc663564..b615bc592 100644
--- a/packages/gooddata-eval/AGENTS.md
+++ b/packages/gooddata-eval/AGENTS.md
@@ -27,7 +27,7 @@ experiment. The newest and most actively developed package in the repo.
 | `core/agentic/` | multi-turn agentic evaluation per capability, **plus** all Langfuse trace polling and linking (`_langfuse.py`, `_trace_linker.py`) |
 | `core/chat/` | SSE client for the agent's streaming chat endpoint |
 | `core/summary/` | HTTP client for the dedicated dashboard-summary endpoint — a single-shot chat backend, not reporting |
-| `core/takeaway/` | HTTP client for the AI Publisher takeaway endpoint — a single-shot backend like `core/summary/`, keeping the whole response on `ChatResult.raw_response` |
+| `core/takeaway/` | HTTP client for the AI Publisher takeaway endpoint — a single-shot backend like `core/summary/`, keeping the whole response on `ChatResult.raw_response` — plus `evaluate_document_takeaway`, the K-run, gate and Langfuse entry point the Tavern shim calls |
 | `core/dataset/` | dataset format, loading, and `from_insights.py` — dataset generation from a workspace's real insights |
 | `core/evaluators/` | single-shot evaluators and their registry |
 | `core/langfuse/` | the whole Langfuse v4 client: `_env` (base URL + credentials), `otlp` (OTLP/JSON encoding), `experiment` (root-span construction, score targets), `item_scope` (an item's experiment scope sent to gen-ai as baggage, and which conversations joined), `observations` (trace reads), `client` (httpx calls), `sink` (single-shot results as experiments) |
diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md
index 702d60cc3..c6df83470 100644
--- a/packages/gooddata-eval/README.md
+++ b/packages/gooddata-eval/README.md
@@ -132,8 +132,8 @@ gd-eval run \
 | `--runs K` | `2` | Independent runs per item. |
 | `--gate` | `any` | Which verdict decides an item: `any` = pass@K (a run passing is enough), `power` = pass^K (every run must pass, so the verdict measures stability). Identical at `--runs 1`. Kinds that repeat K runs only — `power` is refused when the dataset also has non-agentic items or `agentic_conversation`, which are always decided on pass@K. |
 | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests — see *Concurrency and workspace safety* below. |
-| `--judge-model MODEL` | `gpt-4o` | Model used for LLM-as-judge scoring — `agentic_general_question`, `agentic_guardrail`, `general_question`, `guardrail` and `dashboard_summary`. Also settable via `GD_EVAL_JUDGE_MODEL`. Two things to weigh before changing it: the gpt-5 family rejects `temperature=0`, so verdicts stop being reproducible (the run warns when this happens); and choosing the same model the agent runs means the judge grades its own family's output. |
-| `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` items go through the summary endpoint, which has no such option. |
+| `--judge-model MODEL` | `gpt-4o` | Model used for LLM-as-judge scoring — `agentic_general_question`, `agentic_guardrail`, `general_question`, `guardrail`, `dashboard_summary` and `document_takeaway`. Also settable via `GD_EVAL_JUDGE_MODEL`. Two things to weigh before changing it: the gpt-5 family rejects `temperature=0`, so verdicts stop being reproducible (the run warns when this happens); and choosing the same model the agent runs means the judge grades its own family's output. |
+| `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` and `document_takeaway` items go through their own endpoints, which have no such option. |
 
 **Concurrency and workspace safety.** Agentic kinds that create workspace objects
 (`agentic_metric_skill`, `agentic_alert_skill`, `agentic_conversation`, `agentic_kda_skill`) always run one at a
@@ -707,7 +707,7 @@ A dataset is a folder of `.json` files, one per question:
 ```
 
 Supported `test_kind` values: `visualization`, `metric_skill`, `alert_skill`,
-`search_tool`, `general_question`, `guardrail`, `dashboard_summary`.
+`search_tool`, `general_question`, `guardrail`, `dashboard_summary`, `document_takeaway`.
 
 ### `dashboard_summary` items
 
@@ -746,6 +746,64 @@ The `expected_output` rubric:
 Each criterion is scored independently by the LLM judge, so `quality_score`
 is the fraction of satisfied criteria.
 
+### `document_takeaway` items
+
+Takeaway items call the AI Publisher takeaway endpoint
+(`POST /api/v1/ai/workspaces/{ws}/takeaway`) with a `takeaway_input` block, authored in
+snake_case and sent as the endpoint's camelCase body. In a Langfuse dataset the block goes
+in the item's `metadata` (or its `input` object), and `input` can stay a plain case label;
+an item carrying `takeaway_input` is a `document_takeaway` item unless it declares another
+`test_kind`. The endpoint needs the `enableGenAiReportTakeaways` feature flag.
+
+```json
+{
+  "id": "takeaway-001",
+  "dataset_name": "agent_document_takeaway",
+  "test_kind": "document_takeaway",
+  "question": "P01 default format, Sales page, no instructions",
+  "takeaway_input": {
+    "visualizations": ["<visualization id>", {"visualization_id": "<id>", "filters": []}],
+    "filter_context": [],
+    "instructions": {"document": null, "slot": null, "format_hint": null},
+    "max_characters": 300
+  },
+  "expected_output": {
+    "no_data": false,
+    "format": {"no_lists": false},
+    "must_include": ["States one finding that connects at least two of the charts"],
+    "must_not_include": [],
+    "rubric": []
+  }
+}
+```
+
+`takeaway_input` requires `visualizations`; unknown keys are rejected. Every
+`expected_output` key is optional:
+
+- `no_data` — whether the endpoint must answer `noData`; when either side says no data,
+  nothing else is graded.
+- `format` — overrides of the deterministic checks, which gate pass/fail: `bold_headline`,
+  `max_headline_words` (10), `max_sentences` (2), `max_words` (50), `single_paragraph`,
+  `no_lists`, `no_headings`. `false` or `null` switches a check off; unknown keys are
+  rejected. `length` (`max_characters`), `no_ids` and `no_meta` always apply.
+- `default_rules` — `false` drops the default judge criteria (one finding; no executive
+  summary, chart walk-through, background opener, figures, dates, causes the data cannot
+  show, chart description).
+- `must_include`, `must_not_include`, `rubric` — the item's own judge criteria, graded like
+  `dashboard_summary`.
+
+`word_count` and `headline_word_count` are reported in the item detail. The default judge
+criteria are calibrated with `make calibrate-takeaway` (needs `OPENAI_API_KEY`); re-run it
+after changing their wording.
+
+The client retries 429/5xx answers and connection failures. A request error that survives
+the retries is handled differently by the two entry points: `gd-eval run` aborts the item
+with that error, whatever `--runs` is, while the CI entry point
+(`evaluate_document_takeaway`) records it as an errored run, keeps the item's other runs,
+and raises only when no run completed. A read timeout is not retried, and the CI entry
+point records it as an errored run too. Any other 4xx and an unsupported URL scheme are
+never retried and fail the item in both.
+
 ### Items asked in a dashboard context (`user_context`)
 
 An item can be asked the way a user asks from an open dashboard or an attached widget.
@@ -753,7 +811,8 @@ Its `user_context` is sent verbatim as `userContext` on every chat message of th
 follow-up and clarification messages included — the server treats a message without one
 as a cleared context. Requires the `enableAiContextSetup` feature flag on the target
 organization — without it the server ignores the dashboard view. Applies to chat items
-only; `dashboard_summary` items carry their scope in `summary_input` instead. No
+only; `dashboard_summary` and `document_takeaway` items carry their scope in
+`summary_input` / `takeaway_input` instead. No
 `userContext` key is sent at all when neither the item nor, for `agentic_conversation`,
 the current turn sets one.
 
@@ -806,6 +865,7 @@ from that turn on until another turn changes it, as the attached context does in
 | `general_question` | Text answer judged by LLM | `[llm-judge]` |
 | `guardrail` | Refusal/redirect (visualization response auto-fails) | `[llm-judge]` |
 | `dashboard_summary` | Dashboard summary (via `/summary` endpoint) scored against a rubric by LLM | `[llm-judge]` |
+| `document_takeaway` | One short takeaway for a page (via `/takeaway` endpoint): deterministic format checks plus a rubric scored by LLM | `[llm-judge]` |
 
 ## Optional extras
 
diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py
index 4b4b623d8..89a770030 100644
--- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py
+++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py
@@ -7,7 +7,7 @@
 import threading
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import get_args
+from typing import Any, get_args
 
 import httpx
 from gooddata_api_client.exceptions import ApiException, ApiTypeError
@@ -36,32 +36,31 @@
 from gooddata_eval.core.reporting.json_report import build_multi_model_report, write_multi_model_report
 from gooddata_eval.core.runner import ItemReport, run_items
 from gooddata_eval.core.summary.http_client import SummaryClient
+from gooddata_eval.core.takeaway.http_client import TakeawayClient
 from gooddata_eval.core.timing import TIMERS_ENV_VAR
 from gooddata_eval.core.workspace import ModelResolutionError, WorkspaceModelController
 
 _EXIT_OK = 0
 _EXIT_OPERATIONAL_ERROR = 2
-_SUMMARY_TEST_KIND = "dashboard_summary"
 
 
 class _RoutingBackend:
     """Dispatch each item to the right backend by test_kind.
 
-    `dashboard_summary` items go to the dedicated summary endpoint; everything
-    else uses the conversational chat endpoint.
+    Kinds in `by_kind` go to their dedicated endpoint client (`dashboard_summary` to the
+    summary endpoint, `document_takeaway` to the takeaway endpoint); everything else uses
+    the conversational chat endpoint.
     """
 
-    def __init__(self, chat: ChatClient, summary: SummaryClient):
+    def __init__(self, chat: ChatClient, by_kind: dict[str, Any]):
         self._chat = chat
-        self._summary = summary
+        self._by_kind = by_kind
 
     def ask(self, item: DatasetItem) -> ChatResult:
-        if item.test_kind == _SUMMARY_TEST_KIND:
-            return self._summary.ask(item)
-        return self._chat.ask(item)
+        return self._by_kind.get(item.test_kind, self._chat).ask(item)
 
     def close(self) -> None:
-        for backend in (self._chat, self._summary):
+        for backend in (self._chat, *self._by_kind.values()):
             if hasattr(backend, "close"):
                 backend.close()
 
@@ -614,7 +613,14 @@ def on_langfuse_item_done(
                     turn_timeout_s=config.turn_timeout_s,
                     item_timeout_s=config.item_timeout_s,
                 ),
-                SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
+                {
+                    "dashboard_summary": SummaryClient(
+                        host=config.host, token=config.token, workspace_id=config.workspace_id
+                    ),
+                    "document_takeaway": TakeawayClient(
+                        host=config.host, token=config.token, workspace_id=config.workspace_id
+                    ),
+                },
             )
             try:
                 single_report = run_items(
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/takeaway/evaluate.py b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/evaluate.py
new file mode 100644
index 000000000..1ffa3e934
--- /dev/null
+++ b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/evaluate.py
@@ -0,0 +1,275 @@
+# (C) 2026 GoodData Corporation
+"""Entry point for evaluating one document_takeaway item K times, for the Tavern shim.
+
+The CLI evaluates takeaway items through the single-shot runner; CI goes through
+``evaluate_document_takeaway``, which owns the K-run loop, the gate and the Langfuse
+experiment item the same way the ``evaluate_agentic_*`` entry points do, so the nightly
+report reads it unchanged.
+
+The takeaway endpoint returns no conversation id, so there is no gen-ai trace to link:
+every score is written to gd-eval's own experiment span.
+
+A run whose request still fails transiently after the client's retries (5xx, 429, a
+connection failure or a read timeout) is recorded as an errored run and left ungraded,
+like a run the judge could not read; the item raises it only when no run completed. Any
+other 4xx (flag off, no agent, invalid body) or an unsupported URL scheme is a setup
+problem and raises on the first run.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+from gooddata_eval.core.agentic._failed_runs import build_failed_runs
+from gooddata_eval.core.agentic._gate import (
+    DEFAULT_GATE,
+    EvalGate,
+    gate_failure_note,
+    gate_passed,
+    log_gate_scores,
+    stamp_gate_metadata,
+)
+from gooddata_eval.core.agentic._trace_linker import (
+    RunIdentity,
+    RunTraceContext,
+    SubmitTraceLink,
+    open_trace_window,
+    run_trace_link_inline,
+    submit_trace_scoring,
+    utc_now,
+)
+from gooddata_eval.core.config import ReasoningEffort
+from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError
+from gooddata_eval.core.evaluators.base import ItemEvaluation
+from gooddata_eval.core.evaluators.takeaway import DocumentTakeawayEvaluator
+from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, ChatResult, DatasetItem
+from gooddata_eval.core.takeaway.http_client import TakeawayClient
+from gooddata_eval.core.timing import PhaseTimings, sum_timings
+
+_TEST_KIND = "document_takeaway"
+_COUNT_SCORES = ("word_count", "headline_word_count")
+
+
+class TakeawayAssertionError(AgenticAssertionError):
+    """Raised when a document-takeaway evaluation fails its gate."""
+
+
+@dataclass
+class _TakeawayRun:
+    chat_result: ChatResult
+    evaluation: ItemEvaluation
+    latency_s: float
+    timings: PhaseTimings
+    request_error: httpx.HTTPError | None = None
+
+    @property
+    def judge_error(self) -> str | None:
+        return self.evaluation.error
+
+    @property
+    def checks(self) -> dict[str, bool]:
+        return {k: v for k, v in self.evaluation.detail.items() if isinstance(v, bool)}
+
+
+def _dataset_item(dataset_item: dict, dataset_name: str) -> DatasetItem:
+    item = DatasetItem.model_validate(
+        {"id": "", "test_kind": _TEST_KIND, "expected_output": {}, **dataset_item, "dataset_name": dataset_name}
+    )
+    if item.takeaway_input is None:
+        raise ValueError(f"document_takeaway item '{item.id}' is missing required 'takeaway_input'.")
+    return item
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+    if isinstance(exc, httpx.HTTPStatusError):
+        status = exc.response.status_code
+        return status == 429 or status >= 500
+    return isinstance(exc, httpx.TransportError) and not isinstance(exc, httpx.UnsupportedProtocol)
+
+
+def _run_once(client: TakeawayClient, evaluator: DocumentTakeawayEvaluator, item: DatasetItem) -> _TakeawayRun:
+    started = time.monotonic()
+    try:
+        chat_result = client.ask(item)
+    except httpx.HTTPError as exc:
+        if not _is_transient(exc):
+            raise
+        elapsed = time.monotonic() - started
+        error = f"{type(exc).__name__}: {exc}"
+        evaluation = ItemEvaluation(passed=False, rank_key=(-2, 0.0), detail={"request_error": error}, error=error)
+        return _TakeawayRun(ChatResult(), evaluation, elapsed, PhaseTimings(agent_s=elapsed), request_error=exc)
+    asked = time.monotonic()
+    evaluation = evaluator.evaluate(item, chat_result)
+    judged = time.monotonic()
+    return _TakeawayRun(
+        chat_result, evaluation, asked - started, PhaseTimings(agent_s=asked - started, judge_s=judged - asked)
+    )
+
+
+def evaluate_document_takeaway(
+    *,
+    host: str,
+    token: str,
+    workspace_id: str,
+    dataset_item: dict,
+    dataset_name: str,
+    run_timestamp: str | None,
+    model_version_override: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
+    k: int = 1,
+    gate: EvalGate = DEFAULT_GATE,
+    run_metadata_extra: dict | None = None,
+    langfuse: object | None = None,
+    submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    client: TakeawayClient | None = None,
+    evaluator: DocumentTakeawayEvaluator | None = None,
+) -> AgenticEvalOutcome:
+    """Request a takeaway K times, grade each, log to Langfuse, and raise on a failed gate.
+
+    Args:
+        host: GoodData host URL.
+        token: API token.
+        workspace_id: Workspace the visualizations live in.
+        dataset_item: The item as stored: `id`, `question`, `expected_output`, `takeaway_input`.
+        dataset_name: Langfuse dataset name; prefixes the experiment run name.
+        run_timestamp: Shared run timestamp for the run name.
+        model_version_override: Model-version tag for the run; the workspace's active LLM
+            provider otherwise.
+        reasoning_effort: Only names the run: the takeaway endpoint has no reasoning option.
+        k: Runs per item.
+        gate: `any` (pass@K) or `power` (pass^K).
+        run_metadata_extra: Extra experiment-run metadata, e.g. `testing_framework`.
+        langfuse: Langfuse client; built from the LANGFUSE_* environment when None.
+        submit_trace_link: How the Langfuse block is run; inline by default.
+        client: Takeaway client to use instead of one built from host/token/workspace.
+        evaluator: Evaluator to use instead of a default one.
+
+    Raises:
+        TakeawayAssertionError: the gate did not pass.
+        ValueError: the item has no `takeaway_input` or an `expected_output` the evaluator
+            cannot read, raised before any request.
+        httpx.HTTPError: the endpoint refused with a non-transient 4xx, or no run completed;
+            the message carries status and body.
+        JudgeResponseError: the judge returned no readable verdict for any run.
+    """
+    item = _dataset_item(dataset_item, dataset_name)
+    evaluator = evaluator or DocumentTakeawayEvaluator()
+    evaluator.validate(item)
+    own_client = client is None
+    takeaway_client = client or TakeawayClient(host=host, token=token, workspace_id=workspace_id)
+    langfuse, window_start = open_trace_window(langfuse)
+    try:
+        runs = [_run_once(takeaway_client, evaluator, item) for _ in range(k)]
+    finally:
+        if own_client:
+            takeaway_client.close()
+
+    completed = [r for r in runs if r.request_error is None]
+    if not completed:
+        raise runs[-1].request_error or RuntimeError("no takeaway run completed")
+    graded = [r for r in runs if r.judge_error is None]
+    pass_at_k = any(r.evaluation.passed for r in graded)
+    pass_power_k = all(r.evaluation.passed for r in runs)
+
+    if langfuse is not None and item.id:
+        window_end = utc_now()
+
+        def _write_scores(ctx: RunTraceContext) -> None:
+            stamp_gate_metadata(ctx.run_metadata, k=len(runs), gate=gate)
+            for run_idx, run in enumerate(runs):
+                if run.judge_error is not None:
+                    continue
+                with ctx.observe(None, run_idx, output=run.chat_result.raw_response) as tid:
+                    for name, value in run.checks.items():
+                        ctx.score(tid, name=name, value=float(value), data_type="BOOLEAN")
+                    for name in _COUNT_SCORES:
+                        if name in run.evaluation.detail:
+                            ctx.score(tid, name=name, value=run.evaluation.detail[name], data_type="NUMERIC")
+                    ctx.score(
+                        tid, name="document_takeaway_pass", value=float(run.evaluation.passed), data_type="BOOLEAN"
+                    )
+                    log_gate_scores(ctx, tid, gate=gate, pass_at_k=pass_at_k, pass_power_k=pass_power_k)
+                    ctx.quality(tid, strict_checks=run.checks, latency_sec=run.latency_s, cost_usd=None)
+
+        submit_trace_scoring(
+            submit_trace_link,
+            RunIdentity(
+                host,
+                token,
+                workspace_id,
+                dataset_name,
+                run_timestamp,
+                model_version_override,
+                run_metadata_extra,
+                reasoning_effort,
+            ),
+            langfuse=langfuse,
+            dataset_item_id=item.id,
+            conversation_ids=[],
+            window_start=window_start,
+            window_end=window_end,
+            suffix_runs=len(runs) > 1,
+            write_scores=_write_scores,
+            item_input=item.question,
+        )
+
+    item_timings = sum_timings([r.timings for r in runs])
+    unscored = [r.judge_error for r in runs if r.judge_error is not None]
+    request_errors = [r.judge_error for r in runs if r.request_error is not None and r.judge_error is not None]
+    judge_errors = [r.judge_error for r in completed if r.judge_error is not None]
+    best = max(completed, key=lambda r: r.evaluation.rank_key)
+    runs_passed = sum(1 for r in runs if r.evaluation.passed)
+    detail: dict[str, Any] = {
+        **best.evaluation.detail,
+        **({"unscored_runs": len(unscored)} if unscored else {}),
+        **({"judge_errors": judge_errors} if judge_errors else {}),
+        **({"request_errors": request_errors} if request_errors else {}),
+    }
+    failed_runs = build_failed_runs(runs, passed=lambda r: r.evaluation.passed, detail=lambda r: r.evaluation.detail)
+
+    def _attach(exc: TakeawayAssertionError | JudgeResponseError) -> None:
+        exc.reasoning_steps = []
+        exc.conversation_id = ""
+        exc.response_id = None
+        exc.timings = item_timings
+        exc.detail = detail
+        exc.runs_passed = runs_passed
+        exc.runs_effective = len(runs)
+        exc.failed_runs = failed_runs
+
+    if not graded:
+        if request_errors:
+            causes = [
+                *([f"judge returned no readable verdict for {len(judge_errors)} run(s)"] if judge_errors else []),
+                f"request failed for {len(request_errors)} run(s)",
+            ]
+            message = f"no run of {len(runs)} could be graded: {', '.join(causes)}: "
+        else:
+            message = f"judge returned no readable verdict for any of the {len(runs)} run(s): "
+        exc_judge = JudgeResponseError(message + " | ".join(unscored))
+        _attach(exc_judge)
+        raise exc_judge
+
+    if not gate_passed(gate, pass_at_k=pass_at_k, pass_power_k=pass_power_k):
+        failed = [name for name, ok in best.checks.items() if not ok]
+        parts = [
+            "Document takeaway assertion failed.",
+            gate_failure_note(gate, runs_passed, len(runs), len(unscored)),
+            *([f"Failed checks: {', '.join(failed)}."] if failed else []),
+            *([f"Request errors: {' | '.join(request_errors)}"] if request_errors else []),
+            f"Takeaway: {best.chat_result.text_response!r}",
+        ]
+        exc = TakeawayAssertionError(" ".join(parts))
+        _attach(exc)
+        raise exc
+    return AgenticEvalOutcome(
+        detail=detail,
+        timings=item_timings,
+        runs_passed=runs_passed,
+        runs_effective=len(runs),
+        failed_runs=failed_runs,
+    )
diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py
index 8adff1fae..2a53f5cdc 100644
--- a/packages/gooddata-eval/tests/test_cli.py
+++ b/packages/gooddata-eval/tests/test_cli.py
@@ -2,6 +2,7 @@
 import io
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timezone
+from unittest.mock import MagicMock
 
 import httpx
 import orjson
@@ -12,6 +13,7 @@
 from gooddata_eval.core.connection import (
     ConnectionError_,  # noqa: F401 - used in test_cli_operational_error_exits_nonzero
 )
+from gooddata_eval.core.dataset.langfuse_source import _item_from_raw
 from gooddata_eval.core.models import DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 from gooddata_eval.core.timing import TIMERS_ENV_VAR, timers_enabled
@@ -1146,3 +1148,52 @@ def test_no_warning_when_there_are_no_agentic_items(monkeypatch, tmp_path, capsy
     cli_main._warn_if_local_dataset_cannot_link(_local_dataset_config(tmp_path), [])
 
     assert capsys.readouterr().err == ""
+
+
+def _routing_backend() -> tuple[cli_main._RoutingBackend, MagicMock, MagicMock, MagicMock]:
+    chat, summary, takeaway = MagicMock(), MagicMock(), MagicMock()
+    backend = cli_main._RoutingBackend(chat, {"dashboard_summary": summary, "document_takeaway": takeaway})
+    return backend, chat, summary, takeaway
+
+
+def test_cli_routes_takeaway_items_to_takeaway_client() -> None:
+    backend, chat, summary, takeaway = _routing_backend()
+    raw = {
+        "id": "lf-tk-1",
+        "datasetName": "agent_document_takeaway",
+        "input": "P01 default format, Sales page",
+        "metadata": {"takeaway_input": {"visualizations": ["v1", "v2"], "max_characters": 300}},
+        "expectedOutput": {"must_include": ["States one finding"]},
+    }
+    item = _item_from_raw(raw, dataset_name="agent_document_takeaway", test_kind="visualization")
+
+    result = backend.ask(item)
+
+    takeaway.ask.assert_called_once_with(item)
+    assert result is takeaway.ask.return_value
+    chat.ask.assert_not_called()
+    summary.ask.assert_not_called()
+
+
+def test_cli_routes_other_kinds_by_test_kind() -> None:
+    backend, chat, summary, takeaway = _routing_backend()
+    summary_item = DatasetItem(
+        id="s", dataset_name="d", test_kind="dashboard_summary", question="q", expected_output={}
+    )
+    chat_item = DatasetItem(id="c", dataset_name="d", test_kind="general_question", question="q", expected_output="x")
+
+    backend.ask(summary_item)
+    backend.ask(chat_item)
+
+    summary.ask.assert_called_once_with(summary_item)
+    chat.ask.assert_called_once_with(chat_item)
+    takeaway.ask.assert_not_called()
+
+
+def test_routing_backend_closes_every_client() -> None:
+    backend, chat, summary, takeaway = _routing_backend()
+
+    backend.close()
+
+    for client in (chat, summary, takeaway):
+        client.close.assert_called_once_with()
diff --git a/packages/gooddata-eval/tests/test_takeaway_evaluate.py b/packages/gooddata-eval/tests/test_takeaway_evaluate.py
new file mode 100644
index 000000000..112d98df4
--- /dev/null
+++ b/packages/gooddata-eval/tests/test_takeaway_evaluate.py
@@ -0,0 +1,349 @@
+# (C) 2026 GoodData Corporation
+import inspect
+from typing import Any
+from unittest.mock import MagicMock, patch
+
+import httpx
+import pytest
+from gooddata_eval.core.agentic import _langfuse as agentic_langfuse
+from gooddata_eval.core.agentic._trace_linker import run_trace_link_inline
+from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError
+from gooddata_eval.core.evaluators.takeaway import DocumentTakeawayEvaluator
+from gooddata_eval.core.models import ChatResult
+from gooddata_eval.core.takeaway.evaluate import TakeawayAssertionError, evaluate_document_takeaway
+
+from tests._fake_langfuse import FakeLangfuse
+
+_GOOD = "**Growth now comes from new markets.** Sales rose again this period, led by the Nordics, while core markets held flat."
+_LONG = "**Sales grew.** " + " ".join(["word"] * 60) + "."
+_OTLP = "/api/public/otel/v1/traces"
+_SCORES = "/api/public/scores"
+
+
+def _posted_scores(fake_langfuse: FakeLangfuse) -> list[dict]:
+    bodies: list[dict] = []
+    for call in fake_langfuse.calls("POST", _SCORES):
+        body = call["json"]
+        bodies.extend(body if isinstance(body, list) else [body])
+    return bodies
+
+
+def _dataset_item(**overrides: Any) -> dict:
+    return {
+        "id": "item-1",
+        "question": "P01 default format, Sales page",
+        "expected_output": {},
+        "takeaway_input": {"visualizations": ["v1"], "max_characters": 300},
+        **overrides,
+    }
+
+
+def _chat(text: str) -> ChatResult:
+    return ChatResult(text_response=text, raw_response={"takeaway": text, "noData": False, "contextOmitted": []})
+
+
+def _client(*texts: str) -> MagicMock:
+    client = MagicMock()
+    client.ask.side_effect = [_chat(t) for t in texts]
+    return client
+
+
+def _evaluator(*, judge_error: bool = False) -> DocumentTakeawayEvaluator:
+    with patch("openai.OpenAI"), patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}):
+        ev = DocumentTakeawayEvaluator()
+    for judge, verdict in ((ev._positive_judge, True), (ev._violation_judge, False)):
+        judge.score = MagicMock(
+            side_effect=JudgeResponseError("empty body twice") if judge_error else None, return_value=(verdict, "ok")
+        )
+    return ev
+
+
+def _evaluate(client: MagicMock | None, **kwargs: Any) -> Any:
+    params: dict[str, Any] = {
+        "host": "https://h",
+        "token": "tok",
+        "workspace_id": "ws",
+        "dataset_item": _dataset_item(),
+        "dataset_name": "agent_document_takeaway",
+        "run_timestamp": "2026-10-22_00-00-00",
+        "client": client,
+        "evaluator": _evaluator(),
+        "langfuse": MagicMock(),
+        **kwargs,
+    }
+    return evaluate_document_takeaway(**params)
+
+
+def test_evaluate_builds_and_closes_its_own_client_when_none_is_given() -> None:
+    client = _client(_GOOD)
+    with patch("gooddata_eval.core.takeaway.evaluate.TakeawayClient", return_value=client) as client_cls:
+        outcome = _evaluate(None)
+
+    client_cls.assert_called_once_with(host="https://h", token="tok", workspace_id="ws")
+    client.close.assert_called_once_with()
+    assert outcome.runs_passed == 1
+
+
+def test_evaluate_leaves_an_injected_client_open() -> None:
+    client = _client(_GOOD)
+    _evaluate(client)
+
+    client.close.assert_not_called()
+
+
+def test_evaluate_passes_and_returns_the_outcome() -> None:
+    outcome = _evaluate(_client(_GOOD, _GOOD), k=2)
+
+    assert outcome.runs_passed == 2
+    assert outcome.runs_effective == 2
+    assert outcome.detail["word_count"] == 20
+    assert outcome.failed_runs == []
+    assert outcome.timings.agent_s >= 0.0
+
+
+def test_evaluate_raises_on_fail_with_detail() -> None:
+    with pytest.raises(TakeawayAssertionError) as exc_info:
+        _evaluate(_client(_LONG))
+
+    exc = exc_info.value
+    assert "word_budget" in str(exc)
+    assert "Gate pass@1 failed: 0/1 runs passed" in str(exc)
+    assert exc.detail["word_budget"] is False
+    assert exc.detail["word_count"] == 62
+    assert exc.runs_passed == 0
+    assert exc.runs_effective == 1
+    assert len(exc.failed_runs) == 1
+
+
+def test_evaluate_power_gate_fails_on_one_bad_run() -> None:
+    with pytest.raises(TakeawayAssertionError, match="pass\\^2 failed: 1/2"):
+        _evaluate(_client(_GOOD, _LONG), k=2, gate="power")
+
+    outcome = _evaluate(_client(_GOOD, _LONG), k=2, gate="any")
+    assert outcome.runs_passed == 1
+    assert len(outcome.failed_runs) == 1
+
+
+def test_every_run_ungraded_is_a_judge_error_not_a_failure() -> None:
+    with pytest.raises(JudgeResponseError, match="no readable verdict") as exc_info:
+        _evaluate(_client(_GOOD, _GOOD), k=2, evaluator=_evaluator(judge_error=True))
+
+    exc = exc_info.value
+    assert exc.runs_effective == 2
+    assert exc.runs_passed == 0
+    assert exc.detail["actual_output"] == _GOOD
+    assert exc.detail["word_count"] == 20
+    assert exc.detail["unscored_runs"] == 2
+    assert [run["error"] for run in exc.failed_runs] == [exc.detail["judge_errors"][0]] * 2
+
+
+def test_item_without_takeaway_input_is_rejected_before_any_request() -> None:
+    client = _client(_GOOD)
+    item = _dataset_item()
+    del item["takeaway_input"]
+
+    with pytest.raises(ValueError, match="'item-1'.*takeaway_input"):
+        _evaluate(client, dataset_item=item)
+
+    client.ask.assert_not_called()
+
+
+def test_http_error_surfaces_with_status_and_body() -> None:
+    request = httpx.Request("POST", "https://h/api/v1/ai/workspaces/ws/takeaway")
+    response = httpx.Response(400, request=request, text='{"reason": "NO_AVAILABLE_AGENT"}')
+    client = MagicMock()
+    client.ask.side_effect = httpx.HTTPStatusError(
+        f"Takeaway request failed with 400: {response.text}", request=request, response=response
+    )
+    submit = MagicMock()
+
+    with pytest.raises(httpx.HTTPStatusError, match="NO_AVAILABLE_AGENT"):
+        _evaluate(client, submit_trace_link=submit)
+
+    submit.assert_not_called()
+
+
+def test_signature_accepts_what_the_tavern_shim_passes() -> None:
+    params = inspect.signature(evaluate_document_takeaway).parameters
+
+    for name in (
+        "host",
+        "token",
+        "workspace_id",
+        "dataset_item",
+        "dataset_name",
+        "run_timestamp",
+        "model_version_override",
+        "reasoning_effort",
+        "k",
+        "gate",
+        "run_metadata_extra",
+    ):
+        assert name in params, name
+
+
+def test_langfuse_item_has_testing_framework_metadata(fake_langfuse: FakeLangfuse) -> None:
+    with patch.object(agentic_langfuse, "get_model_version", return_value="gpt-5.2"):
+        _evaluate(
+            _client(_GOOD),
+            langfuse=None,
+            submit_trace_link=run_trace_link_inline,
+            run_metadata_extra={"testing_framework": "tavern-e2e", "github_run_id": "123"},
+        )
+
+    spans = [
+        span
+        for call in fake_langfuse.calls("POST", _OTLP)
+        for resource in call["json"]["resourceSpans"]
+        for scope in resource["scopeSpans"]
+        for span in scope["spans"]
+    ]
+    assert len(spans) == 1
+    attrs = {a["key"]: next(iter(a["value"].values())) for a in spans[0]["attributes"]}
+    assert attrs["langfuse.experiment.metadata.testing_framework"] == "tavern-e2e"
+    assert attrs["langfuse.experiment.metadata.github_run_id"] == "123"
+    assert attrs["langfuse.experiment.metadata.model_version"] == "gpt-5.2"
+
+    scores = {score["name"]: score for score in _posted_scores(fake_langfuse)}
+    for name in ("gate_passed", "pass_at_k", "pass_power_k", "word_budget", "no_meta", "quality_score"):
+        assert name in scores, name
+    assert scores["word_count"]["value"] == 20
+    assert scores["word_count"]["dataType"] == "NUMERIC"
+    assert scores["headline_word_count"]["value"] == 6
+    assert scores["default_exclude_exact_figure"]["value"] == 1.0
+
+
+def _http_error(status: int, body: str) -> httpx.HTTPStatusError:
+    request = httpx.Request("POST", "https://h/api/v1/ai/workspaces/ws/takeaway")
+    response = httpx.Response(status, request=request, text=body)
+    return httpx.HTTPStatusError(f"Takeaway request failed with {status}: {body}", request=request, response=response)
+
+
+def test_a_transient_error_on_one_run_keeps_the_graded_runs(fake_langfuse: FakeLangfuse) -> None:
+    client = MagicMock()
+    client.ask.side_effect = [_chat(_GOOD), _chat(_GOOD), _http_error(502, "Visualizations failed to execute.")]
+
+    with patch.object(agentic_langfuse, "get_model_version", return_value="gpt-5.2"):
+        outcome = _evaluate(client, k=3, gate="any", langfuse=None, submit_trace_link=run_trace_link_inline)
+
+    assert outcome.runs_passed == 2
+    assert outcome.runs_effective == 3
+    assert outcome.detail["unscored_runs"] == 1
+    assert outcome.detail["request_errors"] == [outcome.failed_runs[0]["error"]]
+    assert "judge_errors" not in outcome.detail
+    assert len(outcome.failed_runs) == 1
+    assert "502" in outcome.failed_runs[0]["error"]
+    assert "failed to execute" in outcome.failed_runs[0]["error"]
+    gate_scores = [s for s in _posted_scores(fake_langfuse) if s["name"] == "gate_passed"]
+    assert len(gate_scores) == 2
+    assert all(score["value"] == 1 for score in gate_scores)
+
+
+def test_a_transient_error_on_every_run_raises_it() -> None:
+    client = MagicMock()
+    client.ask.side_effect = [_http_error(502, "Visualizations failed to execute.")] * 2
+    submit = MagicMock()
+
+    with pytest.raises(httpx.HTTPStatusError, match="502.*failed to execute"):
+        _evaluate(client, k=2, submit_trace_link=submit)
+
+    submit.assert_not_called()
+
+
+def test_a_deterministic_client_error_raises_on_the_first_run() -> None:
+    client = MagicMock()
+    client.ask.side_effect = [_http_error(400, '{"reason": "NO_AVAILABLE_AGENT"}'), _chat(_GOOD), _chat(_GOOD)]
+
+    with pytest.raises(httpx.HTTPStatusError, match="NO_AVAILABLE_AGENT"):
+        _evaluate(client, k=3)
+
+    assert client.ask.call_count == 1
+
+
+def test_no_data_run_publishes_no_word_counts(fake_langfuse: FakeLangfuse) -> None:
+    client = MagicMock()
+    client.ask.return_value = ChatResult(text_response="", raw_response={"takeaway": None, "noData": True})
+
+    with patch.object(agentic_langfuse, "get_model_version", return_value="gpt-5.2"):
+        _evaluate(
+            client,
+            dataset_item=_dataset_item(expected_output={"no_data": True}),
+            langfuse=None,
+            submit_trace_link=run_trace_link_inline,
+        )
+
+    names = {score["name"] for score in _posted_scores(fake_langfuse)}
+    assert "no_data" in names
+    assert "word_count" not in names
+    assert "headline_word_count" not in names
+
+
+def test_an_invalid_fixture_fails_before_any_request() -> None:
+    client = _client(_GOOD)
+    submit = MagicMock()
+
+    with pytest.raises(ValueError, match="max_sentence"):
+        _evaluate(
+            client,
+            dataset_item=_dataset_item(expected_output={"format": {"max_sentence": 1}}),
+            submit_trace_link=submit,
+        )
+
+    client.ask.assert_not_called()
+    submit.assert_not_called()
+
+
+def test_each_run_writes_its_own_document_takeaway_pass(fake_langfuse: FakeLangfuse) -> None:
+    with patch.object(agentic_langfuse, "get_model_version", return_value="gpt-5.2"):
+        _evaluate(_client(_GOOD, _LONG), k=2, gate="any", langfuse=None, submit_trace_link=run_trace_link_inline)
+
+    scores = [s for s in _posted_scores(fake_langfuse) if s["name"] == "document_takeaway_pass"]
+    assert sorted(score["value"] for score in scores) == [0, 1]
+    assert all(score["dataType"] == "BOOLEAN" for score in scores)
+
+
+def test_a_read_timeout_is_an_errored_run_not_an_item_failure() -> None:
+    client = MagicMock()
+    client.ask.side_effect = [_chat(_GOOD), httpx.ReadTimeout("read timed out")]
+
+    outcome = _evaluate(client, k=2, gate="any")
+
+    assert outcome.runs_passed == 1
+    assert "ReadTimeout" in outcome.detail["request_errors"][0]
+
+
+def test_an_unsupported_protocol_raises_on_the_first_run() -> None:
+    client = MagicMock()
+    client.ask.side_effect = [httpx.UnsupportedProtocol("no scheme"), _chat(_GOOD)]
+
+    with pytest.raises(httpx.UnsupportedProtocol):
+        _evaluate(client, k=2)
+
+    assert client.ask.call_count == 1
+
+
+def test_power_gate_failure_from_an_errored_run_names_the_request_error() -> None:
+    client = MagicMock()
+    client.ask.side_effect = [_chat(_GOOD), _http_error(502, "Visualizations failed to execute.")]
+
+    with pytest.raises(TakeawayAssertionError) as exc_info:
+        _evaluate(client, k=2, gate="power")
+
+    message = str(exc_info.value)
+    assert "Failed checks: none" not in message
+    assert "Request errors:" in message
+    assert "502" in message
+
+
+def test_an_ungraded_item_with_request_errors_says_so() -> None:
+    client = MagicMock()
+    client.ask.side_effect = [_chat(_GOOD), _http_error(502, "Visualizations failed to execute.")]
+
+    with pytest.raises(JudgeResponseError) as exc_info:
+        _evaluate(client, k=2, evaluator=_evaluator(judge_error=True))
+
+    message = str(exc_info.value)
+    assert "no run of 2 could be graded" in message
+    assert "judge returned no readable verdict for 1 run(s)" in message
+    assert "request failed for 1 run(s)" in message
+    assert "502" in message