fix(gooddata-eval): send the run metadata to gen-ai as experiment metadata

packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py

@@ -17,7 +17,6 @@
 import json
 import logging
 import os
-import re
 import time
 from dataclasses import dataclass, field
 from typing import Any, Callable, Iterable, TypeVar
@@ -26,7 +25,14 @@
 import httpx
 
 from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort
-from gooddata_eval.core.langfuse.item_scope import JoinedRun, baggage_entries, current_scope, mark_joined, trace_ids_for
+from gooddata_eval.core.langfuse.item_scope import (
+    BAGGAGE_KEY,
+    JoinedRun,
+    baggage_entries,
+    current_scope,
+    mark_joined,
+    trace_ids_for,
+)
 from gooddata_eval.core.models import ChatResult, DatasetItem
 from gooddata_eval.core.timing import note_trace_latency
 
@@ -151,7 +157,6 @@ def _float_env(name: str, default: float) -> float:
 # a plain token (letters, digits, `_`, `-`); any other key is skipped with a warning, because httpx
 # refuses a non-ASCII header and gen-ai's OTel propagator drops or rewrites the rest.
 _TRACE_LABELS_ENV = "GOODDATA_EVAL_TRACE_LABELS"
-_TRACE_LABEL_KEY = re.compile(r"[A-Za-z0-9_-]+")
 
 
 @functools.cache
@@ -173,7 +178,7 @@ def _trace_baggage() -> dict[str, str]:
         key = key.strip()
         if not sep or not key:
             continue
-        if _TRACE_LABEL_KEY.fullmatch(key):
+        if BAGGAGE_KEY.fullmatch(key):
             labels[key] = value.strip()
         else:
             _warn_skipped_trace_label(key)

packages/gooddata-eval/src/gooddata_eval/core/langfuse/item_scope.py

@@ -14,8 +14,12 @@
 from __future__ import annotations
 
 import contextvars
+import functools
 import hashlib
+import json
+import logging
 import os
+import re
 import threading
 from collections.abc import Iterator
 from contextlib import contextmanager
@@ -38,6 +42,12 @@
 EXPERIMENT_NAME_KEY = "langfuse_experiment_name"
 EXPERIMENT_DATASET_ID_KEY = "langfuse_experiment_dataset_id"
 EXPERIMENT_ITEM_ID_KEY = "langfuse_experiment_item_id"
+EXPERIMENT_METADATA_KEY_PREFIX = "langfuse_experiment_metadata_"
+
+# A key gen-ai's OTel propagator and httpx both pass through unchanged.
+BAGGAGE_KEY = re.compile(r"[A-Za-z0-9_-]+")
+
+_log = logging.getLogger(__name__)
 
 
 def join_enabled() -> bool:
@@ -108,7 +118,12 @@ def trace_ids_for(conversation_id: str) -> tuple[str, str]:
 
 
 def baggage_entries(scope: ItemTraceScope, run_name: str, conversation_id: str) -> list[str]:
-    """The W3C baggage members that put one run's gen-ai spans under its experiment item."""
+    """The W3C baggage members that put one run's gen-ai spans under its experiment item.
+
+    They carry the run metadata known before the run as the experiment metadata. A key that
+    is not a plain token (letters, digits, ``_``, ``-``) is skipped with a warning: httpx
+    refuses a non-ASCII header, and a space, comma or ``=`` would break the baggage entry.
+    """
     trace_id, span_id = trace_ids_for(conversation_id)
     return [
         f"{TRACE_ID_KEY}={trace_id}",
@@ -117,9 +132,32 @@ def baggage_entries(scope: ItemTraceScope, run_name: str, conversation_id: str)
         f"{EXPERIMENT_NAME_KEY}={quote(run_name, safe='')}",
         f"{EXPERIMENT_DATASET_ID_KEY}={quote(scope.dataset_id, safe='')}",
         f"{EXPERIMENT_ITEM_ID_KEY}={quote(scope.item_id, safe='')}",
+        *(
+            f"{EXPERIMENT_METADATA_KEY_PREFIX}{key}={quote(_baggage_value(value), safe='')}"
+            for key, value in scope.run_metadata.items()
+            if value is not None and _is_baggage_key(key)
+        ),
     ]
 
 
+def _is_baggage_key(key: str) -> bool:
+    if BAGGAGE_KEY.fullmatch(key):
+        return True
+    _warn_skipped_metadata_key(key)
+    return False
+
+
+@functools.cache
+def _warn_skipped_metadata_key(key: str) -> None:
+    """Warn once per key per process: the entries are rebuilt on every chat turn."""
+    _log.warning("Run metadata key %r is not sent to gen-ai; keys must match [A-Za-z0-9_-]+", key)
+
+
+def _baggage_value(value: Any) -> str:
+    """``value`` as the root span shows it: a string as is, anything else as JSON (``true``, ``1``, ``["a"]``)."""
+    return value if isinstance(value, str) else json.dumps(value, default=str)
+
+
 @dataclass(frozen=True)
 class JoinedRun:
     """The experiment a joined conversation's gen-ai spans were told they belong to."""

packages/gooddata-eval/tests/test_sse_client.py

@@ -1291,6 +1291,38 @@ def test_run_index_follows_the_order_conversations_are_first_sent(monkeypatch: p
     assert ids == [experiment_id_for(name) for name in names]
 
 
+def test_the_run_metadata_rides_along_as_experiment_metadata(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Without it, a gen-ai span that names the experiment can leave its metadata empty."""
+    scope = ItemTraceScope(
+        base_name="ds_ts_model",
+        suffix_runs=True,
+        dataset_id="ds-1",
+        item_id="item-1",
+        run_metadata={
+            "github_ref_name": "feat/a b",
+            "eval_k": 1,
+            "tags": ["x", "y"],
+            "rerun": True,
+            "unset": None,
+            "ci run,id": "7",
+            "zkouška": "x",
+        },
+    )
+    requests: list[httpx.Request] = []
+    client = _scoped_client(monkeypatch, requests)
+    with item_scope(scope):
+        client.send_message("conv-metadata", "q")
+
+    (baggage,) = _baggage_of(requests)
+    prefix = "langfuse_experiment_metadata_"
+    assert {k[len(prefix) :]: v for k, v in baggage.items() if k.startswith(prefix)} == {
+        "github_ref_name": "feat/a b",
+        "eval_k": "1",
+        "tags": '["x", "y"]',
+        "rerun": "true",
+    }
+
+
 def test_labels_and_scope_share_one_baggage_header(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", "github_run_id=42")
     requests: list[httpx.Request] = []