@@ -156,6 +156,7 @@ Run this before guessing a `--model` string.
 | `GD_EVAL_AGENT_ID` | which agent to drive, same as `--agent-id` |
 | `LANGFUSE_HOST`, `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` | `--langfuse`, `--langfuse-dataset` |
 | `GOODDATA_EVAL_CHAT_*` | SSE retry, backoff and timeout knobs |
+| `GOODDATA_EVAL_TRACE_LABELS` | `key=value,...` labels stamped on every observation of a chat conversation via W3C baggage; `model_version` also sets the observation `version`, so set it only for a one-model run: every `--model` in one process gets the same labels. Keys must match `[A-Za-z0-9_-]+`; others are skipped with a warning. Dashboard-summary runs are not labelled |
 | `GD_EVAL_TIMERS` | same as `--timers` |
 
 A gitignored `.env` at the repo root is the normal place for these; load it with
 
@@ -27,6 +27,7 @@
 from gooddata_eval.core.agentic.what_if import evaluate_agentic_what_if
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError
+from gooddata_eval.core.evaluators.visualization import requires_execution_of
 from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 
@@ -188,6 +189,7 @@ def _dispatch_agentic(
             workspace_id=workspace_id,
             question=item.question,
             expected_outputs=_parse_visualization_expected(eo),
+            requires_execution=requires_execution_of(eo),
             k=k,
             gate=gate,
             agent_id=agent_id,
@@ -230,8 +232,8 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
-            **lf_kw,
             user_context=item.user_context,
+            **lf_kw,
         )
     elif kind == "agentic_alert_skill":
         return evaluate_agentic_alert_skill(
 
@@ -9,6 +9,7 @@
 from dataclasses import dataclass, field
 from typing import Any
 
+from gooddata_api_client.exceptions import NotFoundException
 from gooddata_sdk import GoodDataSdk
 
 from gooddata_eval.core.agentic._failed_runs import build_failed_runs
@@ -51,6 +52,9 @@
 
 _DEFAULT_K = 1
 _DEFAULT_MAX_ITERATIONS = 7
+# Seconds to wait before each check that a deleted metric stayed deleted, so the checks fall
+# 3, 6 and 12 seconds after the delete; see _delete_metric.
+_DELETE_RECHECK_DELAYS_S: tuple[float, ...] = (3.0, 3.0, 6.0)
 
 
 def _best_maql_match(actual_maql: str, expected_outputs: list[dict]) -> tuple[bool, str]:
@@ -239,11 +243,53 @@ def _delete_metric(sdk: GoodDataSdk, workspace_id: str, metric_id: str) -> None:
     MAQL) and the assertion fails. Deleting the created metric on the way out keeps
     the workspace clean for the next run. Best-effort: failures are logged, not raised.
     Mirrors ``alert_skill._delete_alert``.
+
+    A delete does not stay done on its own. ``create_metric`` reads the whole analytics
+    model and writes it back, so a concurrent test whose read preceded this delete
+    restores the metric when its write lands. The metric is therefore checked again after
+    each of ``_DELETE_RECHECK_DELAYS_S`` and deleted again while it is still there.
     """
+    if not _try_delete_metric(sdk, workspace_id, metric_id):
+        return
+    for delay in _DELETE_RECHECK_DELAYS_S:
+        time.sleep(delay)
+        if not _metric_exists(sdk, workspace_id, metric_id):
+            return
+        print(f"[CLEANUP] Metric {metric_id} is back or could not be checked; deleting again")
+        if not _try_delete_metric(sdk, workspace_id, metric_id):
+            return
+    if _DELETE_RECHECK_DELAYS_S:
+        # Every check found it back, so the last delete is unverified: say so rather than
+        # report a clean workspace. The post-run reset is what removes it for good.
+        print(
+            f"[CLEANUP] Metric {metric_id} was back or unchecked at all {len(_DELETE_RECHECK_DELAYS_S)} checks; "
+            "the last delete is not verified and the metric may remain"
+        )
+
+
+def _try_delete_metric(sdk: GoodDataSdk, workspace_id: str, metric_id: str) -> bool:
+    """Delete the metric; True when it is gone afterwards, a 404 included."""
     try:
         sdk._client.entities_api.delete_entity_metrics(workspace_id, metric_id)
+    except NotFoundException:
+        return True
     except Exception as exc:
         print(f"[CLEANUP] Failed to delete metric {metric_id}: {exc}")
+        return False
+    return True
+
+
+def _metric_exists(sdk: GoodDataSdk, workspace_id: str, metric_id: str) -> bool:
+    """Whether the metric may still be in the workspace. Only a 404 reads as absent; any other
+    lookup failure reads as present, so the caller deletes again rather than leave a restored
+    metric behind. The recheck delays bound how often that happens."""
+    try:
+        sdk._client.entities_api.get_entity_metrics(workspace_id, metric_id)
+    except NotFoundException:
+        return False
+    except Exception as exc:
+        print(f"[CLEANUP] Could not check metric {metric_id}: {exc}")
+    return True
 
 
 def _execute_single_metric_run(
 
@@ -38,6 +38,7 @@
     _check_visualization_skill_activated,
     _evaluate_against_candidates,
     evaluation_result_detail,
+    with_execution,
 )
 from gooddata_eval.core.models import (
     AgenticAssertionError,
@@ -195,6 +196,7 @@ def _execute_single_run(
     question: str,
     expected_outputs: list[CreatedVisualization],
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
+    requires_execution: bool = False,
 ) -> RunResult:
     """Drive one full multi-turn conversation and evaluate the result."""
     run_started = time.monotonic()
@@ -221,6 +223,9 @@ def _execute_single_run(
         # K-run already completed along with any exit_reason.
         print(f"[CHAT] send_message failed for conversation {conversation_id}: {exc}")
         current_result = getattr(exc, "partial_result", None) or ChatResult()
+        # The loop that collects events does not run after CHAT_ERROR, so a partial result's
+        # chart executions are added here.
+        all_tool_call_events.extend(current_result.tool_call_events)
         exit_reason = LoopExit.CHAT_ERROR
     # Counted here, not at the top of the loop: this request is sent unconditionally, so
     # with max_iterations=0 the loop body never runs and total_turns would report 0 turns
@@ -272,6 +277,9 @@ def _execute_single_run(
             partial = getattr(exc, "partial_result", None)
             if partial is not None:
                 current_result = partial
+                # The loop adds a turn's events at its next pass, which a break skips. The
+                # partial result is scored, so its chart executions count too.
+                all_tool_call_events.extend(partial.tool_call_events)
             exit_reason = LoopExit.CHAT_ERROR
             break
 
@@ -281,6 +289,7 @@ def _execute_single_run(
         actual_output = current_result.created_visualizations.objects[0]
 
     eval_result, best_expected = _evaluate_against_candidates(expected_outputs, actual_output, skill_activated)
+    eval_result = with_execution(eval_result, all_tool_call_events, current_result.text_response, requires_execution)
 
     return RunResult(
         conversation_id=conversation_id,
@@ -310,13 +319,16 @@ def run_agentic_visualization(
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
     user_context: dict | None = None,
+    requires_execution: bool = False,
 ) -> AgenticRunSummary:
     """Run K independent conversations and return evaluation results.
 
     If initial_conversation_id is provided, Run 0 reuses that conversation
     (e.g. one created by a Tavern YAML POST). Subsequent runs always create
     fresh conversations. Caller-supplied conversations are not deleted; all
     conversations created by this function are deleted on completion.
+
+    ``requires_execution`` fails a run whose agent built the chart but never ran it.
     """
     client = ChatClient(
         host=host,
@@ -331,15 +343,19 @@ def run_agentic_visualization(
     try:
         conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation()
         try:
-            run_results.append(_execute_single_run(client, conv_id_0, question, expected_outputs, max_iterations))
+            run_results.append(
+                _execute_single_run(client, conv_id_0, question, expected_outputs, max_iterations, requires_execution)
+            )
         finally:
             if initial_conversation_id is None:
                 client.delete_conversation(conv_id_0)
 
         for _ in range(1, k):
             conv_id = client.create_conversation()
             try:
-                run_results.append(_execute_single_run(client, conv_id, question, expected_outputs, max_iterations))
+                run_results.append(
+                    _execute_single_run(client, conv_id, question, expected_outputs, max_iterations, requires_execution)
+                )
             finally:
                 client.delete_conversation(conv_id)
     finally:
@@ -410,6 +426,7 @@ def evaluate_agentic_visualization(
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
     user_context: dict | None = None,
+    requires_execution: bool = False,
     gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure.
@@ -433,6 +450,7 @@ def evaluate_agentic_visualization(
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
         user_context=user_context,
+        requires_execution=requires_execution,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -453,13 +471,19 @@ def _write_scores(ctx: RunTraceContext) -> None:
                     "assertion-vis-filters": ev.filters_correct,
                     "assertion-vis-type": ev.viz_type_hard,
                 }
+                if ev.requires_execution:
+                    strict_checks["assertion-vis-executed"] = ev.executed
                 with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid:
                     ctx.score(tid, name="assertion-cross-ref-valid", value=ev.cross_ref_valid, data_type="BOOLEAN")
                     ctx.score(tid, name="assertion-vis-metric", value=ev.metrics_correct, data_type="BOOLEAN")
                     ctx.score(tid, name="assertion-vis-dimensions", value=ev.dimensions_correct, data_type="BOOLEAN")
                     ctx.score(tid, name="assertion-vis-filters", value=ev.filters_correct, data_type="BOOLEAN")
                     ctx.score(tid, name="assertion-vis-type", value=ev.viz_type_hard, data_type="BOOLEAN")
                     ctx.score(tid, name="skill_selection", value=ev.skill_activated, data_type="BOOLEAN")
+                    # Scored on every item, gating or not, so the run rate is visible per model.
+                    ctx.score(tid, name="assertion-vis-executed", value=ev.executed, data_type="BOOLEAN")
+                    if ev.stated_value_matches is not None:
+                        ctx.score(tid, name="stated-value-matches", value=ev.stated_value_matches, data_type="BOOLEAN")
                     # Superseded by log_gate_scores' K-stable names, kept until the readers
                     # migrate: gdc-nas combo_report.py matches on the literal "pass_at_2".
                     ctx.score(tid, name=f"pass_at_{K}", value=summary.pass_at_k, data_type="BOOLEAN")
@@ -562,6 +586,8 @@ def _write_scores(ctx: RunTraceContext) -> None:
             f"    attribute : {ev.filter_attribute_score}\n"
             f"{_filter_diff('attribute', ev)}"
             f"  Viz Type Hard         : {ev.viz_type_hard}\n"
+            f"  Executed              : {ev.executed}{' (required)' if ev.requires_execution else ''}\n"
+            f"  Stated Value Matches  : {ev.stated_value_matches} (reported only)\n"
             "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n",
             reasoning_steps=best.reasoning_steps,
             conversation_id=best.conversation_id,
 
@@ -13,12 +13,15 @@
 protocol, not on this class.
 """
 
+import functools
 import json
 import logging
 import os
+import re
 import time
 from dataclasses import dataclass, field
 from typing import Any, Callable, Iterable, TypeVar
+from urllib.parse import quote
 
 import httpx
 
@@ -135,6 +138,45 @@ def _float_env(name: str, default: float) -> float:
 # 4-turn conversation run to 4x the budget, which is not what a user would sit through.
 _ITEM_TIMEOUT_S = _float_env("GOODDATA_EVAL_CHAT_ITEM_TIMEOUT_S", 0.0)
 
+# Comma-separated `key=value` labels, e.g. `model_version=gpt-5.2,github_run_id=42`, sent as
+# W3C baggage on every chat request. gen-ai's Langfuse span processor copies `langfuse_*`
+# baggage onto each span it starts, so every observation of the conversation -- the cost-bearing
+# generations, retries and the title trace included -- carries them from creation. Langfuse v4
+# cannot add them to an observation afterwards. A value cannot contain a comma, and a key must be
+# a plain token (letters, digits, `_`, `-`); any other key is skipped with a warning, because httpx
+# refuses a non-ASCII header and gen-ai's OTel propagator drops or rewrites the rest.
+_TRACE_LABELS_ENV = "GOODDATA_EVAL_TRACE_LABELS"
+_TRACE_LABEL_KEY = re.compile(r"[A-Za-z0-9_-]+")
+
+
+@functools.cache
+def _warn_skipped_trace_label(key: str) -> None:
+    """Warn once per key per process: every ChatClient re-reads the labels."""
+    _log.warning("%s: skipping label with key %r, keys must match [A-Za-z0-9_-]+", _TRACE_LABELS_ENV, key)
+
+
+def _trace_baggage() -> dict[str, str]:
+    """The `baggage` header for the labels in ``GOODDATA_EVAL_TRACE_LABELS``, or none.
+
+    Each label becomes trace metadata, and ``model_version`` -- the combo the report groups
+    on -- also becomes the observation ``version``, the dimension Langfuse groups cost by.
+    Values are percent-encoded: a space or comma would otherwise end the baggage entry.
+    """
+    labels: dict[str, str] = {}
+    for pair in os.getenv(_TRACE_LABELS_ENV, "").split(","):
+        key, sep, value = pair.partition("=")
+        key = key.strip()
+        if not sep or not key:
+            continue
+        if _TRACE_LABEL_KEY.fullmatch(key):
+            labels[key] = value.strip()
+        else:
+            _warn_skipped_trace_label(key)
+    entries = [f"langfuse_metadata_{k}={quote(v, safe='')}" for k, v in labels.items()]
+    if labels.get("model_version"):
+        entries.insert(0, f"langfuse_version={quote(labels['model_version'], safe='')}")
+    return {"baggage": ",".join(entries)} if entries else {}
+
 
 def set_default_turn_timeout(seconds: float | None) -> None:
     """Set the per-turn budget every ChatClient built afterwards inherits.
@@ -308,11 +350,15 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult:
         # create_adhoc_visualization but the call failed (e.g. data source not
         # accessible). The last attempt is the agent's best answer.
         #
-        # These are raw tool-call arguments, so they carry no `id` -- nothing was
-        # ever persisted. CreatedVisualization requires one, so synthesize a
-        # sentinel rather than letting the whole ChatResult fail to validate:
-        # dropping the turn entirely would score a stalled data source as a
-        # content failure, which is exactly what this fallback exists to prevent.
+        # These are raw tool-call arguments, so they carry no `id` -- the server
+        # mints it. The sentinel marks the chart as never persisted. Logged because
+        # a turn that called the tool and answered without a chart is otherwise
+        # invisible: the user saw no chart either.
+        _log.warning(
+            "create_adhoc_visualization was called %d time(s) but the answer has no visualization part; "
+            "scoring the last call's arguments",
+            len(acc.adhoc_viz_args),
+        )
         payload["createdVisualizations"] = {
             "objects": [{"id": _ADHOC_VIZ_ID, **acc.adhoc_viz_args[-1]}],
             "reasoning": "\n".join(acc.viz_reasoning_parts),
@@ -466,7 +512,7 @@ def __init__(
         the ``enableAiContextSetup`` feature flag is on for the organization.
         """
         self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations"
-        self._auth = {"Authorization": f"Bearer {token}"}
+        self._auth = {"Authorization": f"Bearer {token}", **_trace_baggage()}
         # 0/None disables the cap. Also lowered onto the read timeout: the wall-clock check
         # fires between events, so a turn that goes silent needs the transport to give up too.
         budget = _TURN_TIMEOUT_S if turn_timeout_s is None else turn_timeout_s