From 7fcae3aa9c078c61e60289883cb7b0dc97a08ed8 Mon Sep 17 00:00:00 2001 From: Roman Rakus Date: Tue, 13 Oct 2026 15:15:52 +0200 Subject: [PATCH] feat(gooddata-eval): score v2 Publisher documents The Publisher content v2 model stores pages under `content`, optionally grouped into sections, lays them out as typed widgets inside `row`/`column` containers with `children`, references visualizations as `{identifier, type}`, writes summaries as `takeaway` widgets and states the period as a dashboard date filter. The document-skill evaluator only understood the v1 shape, so a v2 draft failed every structural check. A document with top-level `version: "2"` is now read as v2: pages come from `content`, falling back to the top level when `content` is missing, sections are flattened in order (and win over `pages` if both are present), visualization ids are read as a bare string or as `{identifier}`, and title/heading widgets give the page heading. Only `takeaway` widgets are summaries, and every one on a content page must be written: the default page templates ship an empty `text` widget with id `summary`, while the copilot writes its summaries into takeaways, so that text widget is ignored. An absolute period is compared on `from`/`to`; a relative one cannot match fixed dates and fails the period check with the range it states. Hidden pages (`visible: false`) are out of the viewer, exports and page numbering, so they are left out of everything scored: the page count the part and the tool report (which is what the user is told), the cover/content rule, the chart and summary checks, and the page numbering the judge reads. v1 documents score exactly as before. jira: LX-3336 risk: nonprod Co-Authored-By: Claude Opus 5.5 (1M context) --- .../core/agentic/document_skill.py | 154 ++++++-- .../tests/test_agentic_document_skill.py | 374 ++++++++++++++++-- 2 files changed, 472 insertions(+), 56 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py index 9112f3e02..f6ba747cb 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py @@ -67,6 +67,9 @@ # Template tokens the document renders at export time ({documentName}, {periodStart}, ...). _PLACEHOLDER_RE = re.compile(r"\{\w+\}") _SUMMARY_SLOT = "summary" +_V2 = "2" +_V2_HEADING_WIDGETS = frozenset({"title", "heading"}) +_V2_TAKEAWAY_WIDGET = "takeaway" _NARRATIVE_EVALUATION_STEPS = [ "The actual output is a document rendered as text: its title, period, and per page the heading, " @@ -120,14 +123,44 @@ def legacy_wire_names(parts: list[dict], tool_call_events: list[ToolCallEvent]) return list(dict.fromkeys(found)) +def _is_v2(document: dict) -> bool: + return document.get("version") == _V2 + + +def _pages(document: dict) -> list[Any]: + """Every page in order, hidden ones included. + + v2 keeps them in ``content``, read from the top level when ``content`` is missing, either as + ``pages`` or grouped into ``sections`` that read as one list; ``sections`` wins when both are there. + """ + if not _is_v2(document): + return document.get("pages") or [] + content = document.get("content") + container = content if isinstance(content, dict) else document + sections = container.get("sections") + if isinstance(sections, list): + return [page for section in sections if isinstance(section, dict) for page in section.get("pages") or []] + return container.get("pages") or [] + + +def _visible_pages(document: dict) -> list[Any]: + """The pages a reader sees: a v2 page with ``visible: false`` is out of the viewer, exports and numbering.""" + return [page for page in _pages(document) if not (isinstance(page, dict) and page.get("visible") is False)] + + def _nodes(node: Any) -> Iterator[dict]: - """Every node of a page layout in document order, following ``row`` and ``column`` as gen-ai does.""" + """Every node of a page layout in document order. + + v1 nests through ``row`` and ``column`` lists as gen-ai does; v2 containers hold ``children``. + """ if not isinstance(node, dict): return yield node - for direction in ("row", "column"): - for child in node.get(direction) or []: - yield from _nodes(child) + for key in ("row", "column", "children"): + children = node.get(key) + if isinstance(children, list): + for child in children: + yield from _nodes(child) def _page_kind(page: dict) -> str: @@ -135,14 +168,28 @@ def _page_kind(page: dict) -> str: return str(page.get("kind") or _CONTENT_PAGE) +def _visualization_id(node: dict) -> str | None: + """The node's visualization id: a string in v1 and as-code, ``{"identifier": ...}`` in the v2 API.""" + reference = node.get("visualization") + if isinstance(reference, dict): + reference = reference.get("identifier") + return reference if isinstance(reference, str) else None + + +def _heading(node: dict) -> str | None: + if node.get("type") in _V2_HEADING_WIDGETS: + return _written(node.get("markdown")) + return _written(node.get("heading")) + + def _visualizations_of(document: dict) -> set[str]: - """Every visualization id placed anywhere in the document's page layouts.""" + """Every visualization id placed anywhere in the layouts of the pages a reader sees.""" return { - node["visualization"] - for page in document.get("pages") or [] + visualization + for page in _visible_pages(document) if isinstance(page, dict) for node in _nodes(page.get("layout")) - if isinstance(node.get("visualization"), str) + if (visualization := _visualization_id(node)) is not None } @@ -155,45 +202,82 @@ def _written(text: Any) -> str | None: return text.strip() -def _summary_slot(page: dict) -> dict | None: - """The page's ``summary`` slot, as gen-ai reads it, or ``None`` when the layout has none.""" - for node in _nodes(page.get("layout")): +def _is_takeaway(node: dict) -> bool: + return node.get("type") == _V2_TAKEAWAY_WIDGET + + +def _summary_slots(page: dict) -> list[dict]: + """The page's summaries: every v2 ``takeaway`` widget, or the v1 ``summary`` slot as gen-ai reads it. + + A v2 ``text`` widget with id ``summary`` is static text the default page templates ship empty, + not a summary: the copilot writes its summaries into takeaways. + """ + nodes = list(_nodes(page.get("layout"))) + takeaways = [node for node in nodes if _is_takeaway(node)] + if takeaways: + return takeaways + for node in nodes: if node.get("id") == _SUMMARY_SLOT and "paragraph" in node: - return node - return None + return [node] + return [] def _summary_text(slot: dict) -> str | None: - """The slot's written text: the model's on an AI summary, the typed string on a static one.""" + """The slot's written text: a takeaway's markdown, the model's on an AI summary, the string on a static one.""" + if _is_takeaway(slot): + return _written(slot.get("markdown")) paragraph = slot.get("paragraph") return _written(paragraph.get("text") if isinstance(paragraph, dict) else paragraph) +def _slot_name(slot: dict) -> str: + return "takeaway" if _is_takeaway(slot) else "summary slot" + + def _content_pages(document: dict) -> list[tuple[int, dict]]: - """``(page number, page)`` for every content page.""" + """``(page number, page)`` for every content page a reader sees, numbered as the reader sees them.""" return [ (number, page) - for number, page in enumerate(document.get("pages") or [], start=1) + for number, page in enumerate(_visible_pages(document), start=1) if isinstance(page, dict) and _page_kind(page) == _CONTENT_PAGE ] +def _period(document: dict) -> dict: + period = document.get("period") + return period if isinstance(period, dict) else {} + + +def _period_bounds(document: dict) -> tuple[Any, Any] | None: + """The document's first and last day, or ``None`` for a v2 relative period, which has no fixed dates.""" + period = _period(document) + if not _is_v2(document): + return period.get("start"), period.get("end") + if period.get("type") == "relative": + return None + return period.get("from"), period.get("to") + + +def _period_text(document: dict) -> str: + bounds = _period_bounds(document) + if bounds is None: + period = _period(document) + return f"the relative range {period.get('granularity')} {period.get('from')} to {period.get('to')}" + return f"{bounds[0]} to {bounds[1]}" + + def render_document_text(document: dict) -> str: """The document as the judge reads it: title, period, and per content page its heading, charts and summaries.""" - period = document.get("period") or {} - lines = [f"Document: {document.get('title')}", f"Period: {period.get('start')} to {period.get('end')}"] + lines = [f"Document: {document.get('title')}", f"Period: {_period_text(document)}"] for number, page in _content_pages(document): nodes = list(_nodes(page.get("layout"))) - headings = [h for h in (_written(n.get("heading")) for n in nodes) if h is not None] - charts = [n["visualization"] for n in nodes if isinstance(n.get("visualization"), str)] + headings = [h for h in (_heading(n) for n in nodes) if h is not None] + charts = [c for c in (_visualization_id(n) for n in nodes) if c is not None] lines.append("") lines.append(f"Page {number}: {', '.join(headings) or '(no heading)'}") if charts: lines.append(f"Charts: {', '.join(charts)}") - slot = _summary_slot(page) - text = _summary_text(slot) if slot is not None else None - if text is not None: - lines.append(f"Summary: {text}") + lines.extend(f"Summary: {text}" for text in map(_summary_text, _summary_slots(page)) if text is not None) return "\n".join(lines) @@ -441,7 +525,9 @@ def _score_response( if not ref_matches: failures.append(f"the {_PART_TYPE!r} part shows {part_ref!r}, but {_DRAFT_TOOL} returned {tool_ref!r}") - pages = document.get("pages") or [] + # page_count is what the user is told ("a document with 3 pages"), so it counts only the + # pages a reader sees: a hidden page is out of the viewer, exports and page numbering. + pages = _visible_pages(document) part_count, tool_count = document_part.get("page_count"), tool_result.get("page_count") if part_count != len(pages) or tool_count != len(pages): failures.append( @@ -466,11 +552,10 @@ def _score_response( period_correct = False if applies.period: expected_period = expected_output["period"] - actual_period = document.get("period") or {} - period_correct = all(actual_period.get(key) == expected_period.get(key) for key in ("start", "end")) + period_correct = _period_bounds(document) == (expected_period.get("start"), expected_period.get("end")) if not period_correct: failures.append( - f"the document covers {actual_period.get('start')} to {actual_period.get('end')}, " + f"the document covers {_period_text(document)}, " f"expected {expected_period.get('start')} to {expected_period.get('end')}" ) @@ -483,13 +568,16 @@ def _score_response( summaries_present = False if applies.narrative: - slots = [ - (page, slot) for _number, page in _content_pages(document) if (slot := _summary_slot(page)) is not None + slots = [(page, slot) for _number, page in _content_pages(document) for slot in _summary_slots(page)] + unwritten = [ + f"page {page.get('id')!r} has a {_slot_name(slot)} with no written text" + for page, slot in slots + if _summary_text(slot) is None ] - unwritten = [page for page, slot in slots if _summary_text(slot) is None] if not slots: - failures.append("the document has no summary slot") - failures.extend(f"page {page.get('id')!r} has a summary slot with no written text" for page in unwritten) + missing = "takeaway" if _is_v2(document) else "summary slot" + failures.append(f"the document has no {missing}") + failures.extend(dict.fromkeys(unwritten)) summaries_present = bool(slots) and not unwritten return DocumentEvaluation( diff --git a/packages/gooddata-eval/tests/test_agentic_document_skill.py b/packages/gooddata-eval/tests/test_agentic_document_skill.py index baa60b1de..74f1578c9 100644 --- a/packages/gooddata-eval/tests/test_agentic_document_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_document_skill.py @@ -20,6 +20,7 @@ evaluate_agentic_document_skill, evaluate_document_response, legacy_wire_names, + render_document_text, run_agentic_document_skill, ) from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError @@ -94,6 +95,124 @@ def _document_part( } +# v2 shapes follow the Publisher content v2 model: a top-level `version`, an absolute or relative +# `period`, and `content` holding the pages, whose layout nests `row` and `column` containers +# through `children` down to typed widgets. +_V2_PERIOD = {"type": "absolute", "granularity": "GDC.time.date", "from": "2026-01-01", "to": "2026-06-30"} + + +def _v2_cover_page() -> dict: + return { + "id": "page1", + "kind": "cover", + "layout": { + "type": "column", + "children": [{"type": "title", "id": "coverTitle", "proportion": 2, "markdown": "{documentName}"}], + }, + } + + +def _v2_content_page(page_id: str, *visualizations: str, takeaway: str | None = None, bare_ids: bool = False) -> dict: + """A content page; ``takeaway`` adds a takeaway widget, and ``""`` adds one with no text written.""" + widgets: list[dict] = [ + { + "type": "visualization", + "id": f"widget{i}", + "visualization": v if bare_ids else {"identifier": v, "type": "insight"}, + "title": {"visible": True}, + } + for i, v in enumerate(visualizations, start=1) + ] + row: list[dict] = [{"type": "row", "proportion": 2, "children": widgets}] + if takeaway is not None: + row.append( + { + "type": "takeaway", + "id": "takeaway", + "title": {"text": "Key takeaway"}, + **({"markdown": takeaway} if takeaway else {}), + "generation": {"context": {"widgets": ["widget1"]}}, + } + ) + return { + "id": page_id, + "kind": "content", + "layout": { + "type": "column", + "children": [ + {"type": "title", "id": "pageTitle", "proportion": 2, "markdown": "Revenue"}, + {"type": "row", "proportion": 9, "children": row}, + { + "type": "text", + "id": "footerPageNumber", + "markdown": "{currentPageNumber} / {totalPages}", + "decorative": True, + }, + ], + }, + } + + +def _v2_document(pages: list[dict], *, period: Any = None, form: str = "entity") -> dict: + """A v2 document: ``entity`` keeps the pages in ``content``, ``flat`` lifts the content keys to + the top level, and ``sections`` groups the cover and the remaining pages into two sections.""" + content: dict[str, Any] = {"pages": pages} + if form == "sections": + content = { + "sections": [ + {"id": "section_intro", "title": "Intro", "pages": pages[:1]}, + {"id": "section_body", "pages": pages[1:]}, + ] + } + document: dict[str, Any] = { + "type": "document", + "id": "sales_overview", + "version": "2", + "title": "Sales overview", + "period": period if period is not None else _V2_PERIOD, + } + return {**document, **content} if form == "flat" else {**document, "content": content} + + +def _v2_document_part( + pages: list[dict] | None = None, *, period: Any = None, form: str = "entity", **kwargs: Any +) -> dict: + """A v2 part; ``page_count`` defaults to the visible pages, as a reader counts them.""" + pages = pages if pages is not None else [_v2_cover_page(), _v2_content_page("page2", _REVENUE_TREND)] + kwargs.setdefault("page_count", sum(1 for page in pages if page.get("visible") is not False)) + part = _document_part(pages, document=_v2_document(pages, period=period, form=form), **kwargs) + return {**part, "format": "aac-v2"} + + +# Builders keyed by schema version, for the tests whose assertion is the same for v1 and v2. +_VERSIONS = ["v1", "v2"] + + +def _versioned_cover(version: str) -> dict: + return _cover_page() if version == "v1" else _v2_cover_page() + + +def _versioned_content(version: str, page_id: str, *visualizations: str, summary: str | None = None) -> dict: + if version == "v1": + return ( + _content_page(page_id, *visualizations) + if summary is None + else _summary_page(page_id, summary, *visualizations) + ) + return _v2_content_page(page_id, *visualizations, takeaway=summary) + + +def _versioned_part( + version: str, pages: list[dict] | None = None, *, period: tuple[str, str] | None = None, **kwargs: Any +) -> dict: + if version == "v1": + return _document_part( + pages, period=None if period is None else {"start": period[0], "end": period[1]}, **kwargs + ) + v2_period = None if period is None else {**_V2_PERIOD, "from": period[0], "to": period[1]} + return _v2_document_part(pages, period=v2_period, **kwargs) + + def _draft_result(*, ref: str = "document_1", page_count: int = 2, summaries_from_data: int = 1) -> dict: return { "status": "success", @@ -181,8 +300,9 @@ def _evaluate( ) -def test_a_drafted_document_returned_in_the_chat_item_passes() -> None: - evaluation = _evaluate(_document_part()) +@pytest.mark.parametrize("version", _VERSIONS) +def test_a_drafted_document_returned_in_the_chat_item_passes(version: str) -> None: + evaluation = _evaluate(_versioned_part(version)) assert evaluation.strict_checks == { "document_drafted": True, "document_part_present": True, @@ -196,9 +316,10 @@ def test_a_drafted_document_returned_in_the_chat_item_passes() -> None: assert evaluation.failures == [] -def test_period_and_charts_are_scored_only_when_the_fixture_states_them() -> None: +@pytest.mark.parametrize("version", _VERSIONS) +def test_period_and_charts_are_scored_only_when_the_fixture_states_them(version: str) -> None: expected = {"period": _PERIOD, "visualizations": [{"id": _REVENUE_TREND, "title": "Revenue trend"}]} - checks = _evaluate(_document_part(), expected=expected).strict_checks + checks = _evaluate(_versioned_part(version), expected=expected).strict_checks assert checks["document_period_correct"] is True assert checks["document_charts_matched"] is True @@ -265,19 +386,19 @@ def test_a_new_draft_must_not_edit_a_saved_document() -> None: ] -def test_a_wrong_period_fails() -> None: - evaluation = _evaluate( - _document_part(period={"start": "2025-07-01", "end": "2025-12-31"}), expected={"period": _PERIOD} - ) +@pytest.mark.parametrize("version", _VERSIONS) +def test_a_wrong_period_fails(version: str) -> None: + evaluation = _evaluate(_versioned_part(version, period=("2025-07-01", "2025-12-31")), expected={"period": _PERIOD}) assert evaluation.strict_checks["document_period_correct"] is False assert evaluation.failures == [ "the document covers 2025-07-01 to 2025-12-31, expected 2026-01-01 to 2026-06-30", ] -def test_a_missing_chart_fails_and_is_named() -> None: +@pytest.mark.parametrize("version", _VERSIONS) +def test_a_missing_chart_fails_and_is_named(version: str) -> None: expected = {"visualizations": [{"id": _RETURNS_BY_CATEGORY, "title": "Returns by category"}]} - evaluation = _evaluate(_document_part(), expected=expected) + evaluation = _evaluate(_versioned_part(version), expected=expected) assert evaluation.strict_checks["document_charts_matched"] is False assert evaluation.failures == ["the document does not show chart 'returns_by_category' ('Returns by category')"] @@ -288,9 +409,11 @@ def test_a_routing_miss_fails_the_skill_check_alone() -> None: assert [name for name, ok in evaluation.strict_checks.items() if not ok] == ["document_skill_activated"] -def test_visualizations_are_found_however_deep_the_layout_nests_them() -> None: - page = _content_page("page2", _REVENUE_TREND, _RETURNS_BY_CATEGORY) - assert _visualizations_of({"pages": [_cover_page(), page]}) == {_REVENUE_TREND, _RETURNS_BY_CATEGORY} +@pytest.mark.parametrize("version", _VERSIONS) +def test_visualizations_are_found_however_deep_the_layout_nests_them(version: str) -> None: + page = _versioned_content(version, "page2", _REVENUE_TREND, _RETURNS_BY_CATEGORY) + document = _versioned_part(version, [_versioned_cover(version), page])["document"] + assert _visualizations_of(document) == {_REVENUE_TREND, _RETURNS_BY_CATEGORY} def _legacy_part() -> dict: @@ -597,8 +720,8 @@ def test_a_structured_clarifying_question_counts_as_asking() -> None: # ── page kinds, fixture keys, asking, two drafts in one turn ─────────────── -def _page(page_id: str, kind: str | None) -> dict: - page = _content_page(page_id, _REVENUE_TREND) +def _page(page_id: str, kind: str | None, version: str = "v1") -> dict: + page = _versioned_content(version, page_id, _REVENUE_TREND) if kind is None: del page["kind"] else: @@ -606,14 +729,17 @@ def _page(page_id: str, kind: str | None) -> dict: return page -def test_a_document_that_does_not_open_with_a_cover_fails() -> None: - evaluation = _evaluate(_document_part([_page("page1", "content"), _page("page2", "content")])) +@pytest.mark.parametrize("version", _VERSIONS) +def test_a_document_that_does_not_open_with_a_cover_fails(version: str) -> None: + pages = [_page("page1", "content", version), _page("page2", "content", version)] + evaluation = _evaluate(_versioned_part(version, pages)) assert evaluation.strict_checks["document_pages_consistent"] is False assert evaluation.failures == ["the document opens with a 'content' page, not a cover"] -def test_a_document_without_a_content_page_fails() -> None: - evaluation = _evaluate(_document_part([_cover_page(), _page("page2", "section")])) +@pytest.mark.parametrize("version", _VERSIONS) +def test_a_document_without_a_content_page_fails(version: str) -> None: + evaluation = _evaluate(_versioned_part(version, [_versioned_cover(version), _page("page2", "section", version)])) assert evaluation.strict_checks["document_pages_consistent"] is False assert evaluation.failures == ["the document has no content page"] @@ -780,10 +906,10 @@ def score(self, input: str, expected_output: str, actual_output: str) -> tuple[b return self._passed, "fine" if self._passed else "the summaries ignore returns" -def _narrative_turn(*pages: dict) -> ChatResult: +def _narrative_turn(*pages: dict, version: str = "v1") -> ChatResult: return _chat_result( tool_calls=[_tool_call("draft_document", _draft_result(page_count=len(pages)))], - parts=[_document_part(list(pages))], + parts=[_versioned_part(version, list(pages))], text="I've put together a document.", ) @@ -821,9 +947,14 @@ def test_summaries_are_not_checked_unless_the_fixture_asks_for_the_narrative() - assert "document_narrative_judged" not in evaluation.strict_checks -def test_the_judge_reads_the_document_as_text() -> None: +@pytest.mark.parametrize("version", _VERSIONS) +def test_the_judge_reads_the_document_as_text(version: str) -> None: judge = _FakeJudge() - turn = _narrative_turn(_cover_page(), _summary_page("page2", "Revenue grew 12%.", _REVENUE_TREND)) + turn = _narrative_turn( + _versioned_cover(version), + _versioned_content(version, "page2", _REVENUE_TREND, summary="Revenue grew 12%."), + version=version, + ) run = _execute_single_document_run( _ScriptedChatClient([turn]), "c", "Document on revenue", {"narrative": _NARRATIVE}, 4, judge=judge ) @@ -971,6 +1102,203 @@ def test_only_content_pages_count_for_summaries() -> None: assert evaluation.failures == ["the document has no summary slot"] +# ── v2 documents ──────────────────────────────────────────────────────────── + +_V2_EXPECTED = { + "period": _PERIOD, + "visualizations": [{"id": _REVENUE_TREND, "title": "Revenue trend"}], + "narrative": _NARRATIVE, +} + + +def _v2_written_pages() -> list[dict]: + return [_v2_cover_page(), _v2_content_page("page2", _REVENUE_TREND, takeaway="Revenue grew 12%.")] + + +@pytest.mark.parametrize("form", ["entity", "flat", "sections"]) +def test_a_v2_document_scores_on_every_check_whatever_form_holds_its_pages(form: str) -> None: + evaluation = _evaluate(_v2_document_part(_v2_written_pages(), form=form), expected=_V2_EXPECTED) + assert evaluation.failures == [] + assert evaluation.strict_checks["document_pages_consistent"] is True + assert evaluation.strict_checks["document_period_correct"] is True + assert evaluation.strict_checks["document_charts_matched"] is True + assert evaluation.strict_checks["document_summaries_present"] is True + + +def test_sections_are_read_in_order_as_one_list_of_pages() -> None: + pages = [ + _v2_cover_page(), + _v2_content_page("page2", _REVENUE_TREND), + _v2_content_page("page3", _RETURNS_BY_CATEGORY), + ] + document = _v2_document(pages, form="sections") + assert _visualizations_of(document) == {_REVENUE_TREND, _RETURNS_BY_CATEGORY} + assert render_document_text(document).splitlines()[3:] == [ + "Page 2: Revenue", + "Charts: revenue_trend", + "", + "Page 3: Revenue", + "Charts: returns_by_category", + ] + + +def test_a_v2_document_that_does_not_open_with_a_cover_fails_across_sections() -> None: + pages = [_v2_content_page("page1", _REVENUE_TREND), _v2_cover_page()] + evaluation = _evaluate(_v2_document_part(pages, form="sections")) + assert evaluation.failures == ["the document opens with a 'content' page, not a cover"] + + +def test_a_bare_visualization_id_is_matched_as_in_as_code() -> None: + pages = [_v2_cover_page(), _v2_content_page("page2", _REVENUE_TREND, bare_ids=True)] + expected = {"visualizations": [{"id": _REVENUE_TREND, "title": "Revenue trend"}]} + assert _evaluate(_v2_document_part(pages), expected=expected).strict_checks["document_charts_matched"] is True + + +def test_a_relative_period_never_matches_the_expected_dates() -> None: + relative = {"type": "relative", "granularity": "GDC.time.month", "from": -6, "to": -1} + part = _v2_document_part(period=relative) + evaluation = _evaluate(part, expected={"period": _PERIOD}) + assert evaluation.strict_checks["document_period_correct"] is False + assert evaluation.failures == [ + "the document covers the relative range GDC.time.month -6 to -1, expected 2026-01-01 to 2026-06-30" + ] + assert render_document_text(part["document"]).splitlines()[1] == ( + "Period: the relative range GDC.time.month -6 to -1" + ) + + +def test_a_period_that_is_not_an_object_fails_instead_of_crashing() -> None: + part = _v2_document_part() + part["document"]["period"] = "last six months" + evaluation = _evaluate(part, expected={"period": _PERIOD}) + assert evaluation.strict_checks["document_period_correct"] is False + assert render_document_text(part["document"]).splitlines()[1] == "Period: None to None" + + +def test_every_takeaway_needs_written_text() -> None: + pages = [*_v2_written_pages(), _v2_content_page("page3", _RETURNS_BY_CATEGORY, takeaway="")] + evaluation = _evaluate( + _v2_document_part(pages), expected={"narrative": _NARRATIVE}, tool=_draft_result(page_count=3) + ) + assert evaluation.strict_checks["document_summaries_present"] is False + assert evaluation.failures == ["page 'page3' has a takeaway with no written text"] + + +def test_a_takeaway_made_only_of_placeholders_is_not_written() -> None: + pages = [_v2_cover_page(), _v2_content_page("page2", _REVENUE_TREND, takeaway="{periodStart} – {periodEnd}")] + evaluation = _evaluate(_v2_document_part(pages), expected={"narrative": _NARRATIVE}) + assert evaluation.failures == ["page 'page2' has a takeaway with no written text"] + + +def _with_summary_text(page: dict, markdown: str | None) -> dict: + """``page`` plus the default template's ``text`` widget with id ``summary``, which is not a takeaway.""" + page["layout"]["children"].append( + {"type": "text", "id": "summary", **({"markdown": markdown} if markdown is not None else {})} + ) + return page + + +def test_a_v2_document_without_a_takeaway_has_no_summaries() -> None: + page = _with_summary_text(_v2_content_page("page2", _REVENUE_TREND), "Revenue grew 12%.") + part = _v2_document_part([_v2_cover_page(), page]) + evaluation = _evaluate(part, expected={"narrative": _NARRATIVE}) + assert evaluation.strict_checks["document_summaries_present"] is False + assert evaluation.failures == ["the document has no takeaway"] + assert "Summary:" not in render_document_text(part["document"]) + + +def test_an_empty_summary_text_beside_a_written_takeaway_is_ignored() -> None: + page = _with_summary_text(_v2_content_page("page2", _REVENUE_TREND, takeaway="Revenue grew 12%."), None) + part = _v2_document_part([_v2_cover_page(), page]) + evaluation = _evaluate(part, expected={"narrative": _NARRATIVE}) + assert evaluation.strict_checks["document_summaries_present"] is True + assert render_document_text(part["document"]).splitlines()[-1] == "Summary: Revenue grew 12%." + + +def test_a_written_summary_text_does_not_stand_in_for_an_empty_takeaway() -> None: + page = _with_summary_text(_v2_content_page("page2", _REVENUE_TREND, takeaway=""), "Revenue grew 12%.") + evaluation = _evaluate(_v2_document_part([_v2_cover_page(), page]), expected={"narrative": _NARRATIVE}) + assert evaluation.strict_checks["document_summaries_present"] is False + assert evaluation.failures == ["page 'page2' has a takeaway with no written text"] + + +def test_every_takeaway_on_a_page_is_checked() -> None: + page = _v2_content_page("page2", _REVENUE_TREND, takeaway="Revenue grew 12%.") + page["layout"]["children"][1]["children"].append({"type": "takeaway", "id": "takeaway2"}) + evaluation = _evaluate(_v2_document_part([_v2_cover_page(), page]), expected={"narrative": _NARRATIVE}) + assert evaluation.strict_checks["document_summaries_present"] is False + assert evaluation.failures == ["page 'page2' has a takeaway with no written text"] + + +def test_a_v2_heading_widget_is_a_page_heading() -> None: + page = _v2_content_page("page2", _REVENUE_TREND) + page["layout"]["children"][0] = {"type": "heading", "id": "pageHeading", "markdown": "Returns", "level": 1} + document = _v2_document([_v2_cover_page(), page]) + assert render_document_text(document).splitlines()[3] == "Page 2: Returns" + + +def test_sections_win_over_pages_when_a_document_has_both() -> None: + document = _v2_document([_v2_cover_page(), _v2_content_page("page2", _REVENUE_TREND)], form="sections") + document["content"]["pages"] = [_v2_cover_page(), _v2_content_page("page9", _RETURNS_BY_CATEGORY)] + assert _visualizations_of(document) == {_REVENUE_TREND} + + +def test_a_layout_whose_children_is_not_a_list_is_read_as_a_leaf() -> None: + page = _v2_content_page("page2", _REVENUE_TREND) + page["layout"]["children"].append({"type": "row", "children": 5}) + assert _visualizations_of(_v2_document([_v2_cover_page(), page])) == {_REVENUE_TREND} + + +def _hidden(page: dict) -> dict: + return {**page, "visible": False} + + +def _pages_with_a_hidden_one() -> list[dict]: + return [ + _v2_cover_page(), + _hidden(_v2_content_page("page2", _RETURNS_BY_CATEGORY, takeaway="")), + _v2_content_page("page3", _REVENUE_TREND, takeaway="Revenue grew 12%."), + ] + + +def test_a_hidden_page_is_left_out_of_every_check_and_of_the_judge_text() -> None: + part = _v2_document_part(_pages_with_a_hidden_one()) + evaluation = _evaluate(part, expected={"narrative": _NARRATIVE}, tool=_draft_result(page_count=2)) + assert evaluation.failures == [] + assert evaluation.strict_checks["document_pages_consistent"] is True + assert evaluation.strict_checks["document_summaries_present"] is True + assert _visualizations_of(part["document"]) == {_REVENUE_TREND} + assert render_document_text(part["document"]) == ( + "Document: Sales overview\n" + "Period: 2026-01-01 to 2026-06-30\n" + "\n" + "Page 2: Revenue\n" + "Charts: revenue_trend\n" + "Summary: Revenue grew 12%." + ) + + +def test_a_page_count_that_includes_a_hidden_page_fails() -> None: + evaluation = _evaluate( + _v2_document_part(_pages_with_a_hidden_one(), page_count=3), tool=_draft_result(page_count=3) + ) + assert evaluation.strict_checks["document_pages_consistent"] is False + assert evaluation.failures == ["the document has 2 page(s), but the part says 3 and draft_document said 3"] + + +def test_a_hidden_cover_does_not_open_the_document() -> None: + pages = [_hidden(_v2_cover_page()), _v2_content_page("page2", _REVENUE_TREND)] + evaluation = _evaluate(_v2_document_part(pages), tool=_draft_result(page_count=1)) + assert evaluation.strict_checks["document_pages_consistent"] is False + assert evaluation.failures == ["the document opens with a 'content' page, not a cover"] + + +def test_a_document_whose_only_content_page_is_hidden_has_no_content_page() -> None: + pages = [_v2_cover_page(), _hidden(_v2_content_page("page2", _REVENUE_TREND))] + evaluation = _evaluate(_v2_document_part(pages), tool=_draft_result(page_count=1)) + assert evaluation.failures == ["the document has no content page"] + + def test_the_report_skill_module_still_exports_the_old_names() -> None: with pytest.warns(DeprecationWarning, match="gooddata_eval.core.agentic.document_skill"): report_skill = importlib.reload(importlib.import_module("gooddata_eval.core.agentic.report_skill"))