diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index ad58382e0..3efe7601f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -17,7 +17,6 @@ import json import logging import os -import re import time from dataclasses import dataclass, field from typing import Any, Callable, Iterable, TypeVar @@ -26,7 +25,14 @@ import httpx from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort -from gooddata_eval.core.langfuse.item_scope import JoinedRun, baggage_entries, current_scope, mark_joined, trace_ids_for +from gooddata_eval.core.langfuse.item_scope import ( + BAGGAGE_KEY, + JoinedRun, + baggage_entries, + current_scope, + mark_joined, + trace_ids_for, +) from gooddata_eval.core.models import ChatResult, DatasetItem from gooddata_eval.core.timing import note_trace_latency @@ -151,7 +157,6 @@ def _float_env(name: str, default: float) -> float: # a plain token (letters, digits, `_`, `-`); any other key is skipped with a warning, because httpx # refuses a non-ASCII header and gen-ai's OTel propagator drops or rewrites the rest. _TRACE_LABELS_ENV = "GOODDATA_EVAL_TRACE_LABELS" -_TRACE_LABEL_KEY = re.compile(r"[A-Za-z0-9_-]+") @functools.cache @@ -173,7 +178,7 @@ def _trace_baggage() -> dict[str, str]: key = key.strip() if not sep or not key: continue - if _TRACE_LABEL_KEY.fullmatch(key): + if BAGGAGE_KEY.fullmatch(key): labels[key] = value.strip() else: _warn_skipped_trace_label(key) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/langfuse/item_scope.py b/packages/gooddata-eval/src/gooddata_eval/core/langfuse/item_scope.py index ae68ae712..2d7f25dac 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/langfuse/item_scope.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/langfuse/item_scope.py @@ -14,8 +14,12 @@ from __future__ import annotations import contextvars +import functools import hashlib +import json +import logging import os +import re import threading from collections.abc import Iterator from contextlib import contextmanager @@ -38,6 +42,12 @@ EXPERIMENT_NAME_KEY = "langfuse_experiment_name" EXPERIMENT_DATASET_ID_KEY = "langfuse_experiment_dataset_id" EXPERIMENT_ITEM_ID_KEY = "langfuse_experiment_item_id" +EXPERIMENT_METADATA_KEY_PREFIX = "langfuse_experiment_metadata_" + +# A key gen-ai's OTel propagator and httpx both pass through unchanged. +BAGGAGE_KEY = re.compile(r"[A-Za-z0-9_-]+") + +_log = logging.getLogger(__name__) def join_enabled() -> bool: @@ -108,7 +118,12 @@ def trace_ids_for(conversation_id: str) -> tuple[str, str]: def baggage_entries(scope: ItemTraceScope, run_name: str, conversation_id: str) -> list[str]: - """The W3C baggage members that put one run's gen-ai spans under its experiment item.""" + """The W3C baggage members that put one run's gen-ai spans under its experiment item. + + They carry the run metadata known before the run as the experiment metadata. A key that + is not a plain token (letters, digits, ``_``, ``-``) is skipped with a warning: httpx + refuses a non-ASCII header, and a space, comma or ``=`` would break the baggage entry. + """ trace_id, span_id = trace_ids_for(conversation_id) return [ f"{TRACE_ID_KEY}={trace_id}", @@ -117,9 +132,32 @@ def baggage_entries(scope: ItemTraceScope, run_name: str, conversation_id: str) f"{EXPERIMENT_NAME_KEY}={quote(run_name, safe='')}", f"{EXPERIMENT_DATASET_ID_KEY}={quote(scope.dataset_id, safe='')}", f"{EXPERIMENT_ITEM_ID_KEY}={quote(scope.item_id, safe='')}", + *( + f"{EXPERIMENT_METADATA_KEY_PREFIX}{key}={quote(_baggage_value(value), safe='')}" + for key, value in scope.run_metadata.items() + if value is not None and _is_baggage_key(key) + ), ] +def _is_baggage_key(key: str) -> bool: + if BAGGAGE_KEY.fullmatch(key): + return True + _warn_skipped_metadata_key(key) + return False + + +@functools.cache +def _warn_skipped_metadata_key(key: str) -> None: + """Warn once per key per process: the entries are rebuilt on every chat turn.""" + _log.warning("Run metadata key %r is not sent to gen-ai; keys must match [A-Za-z0-9_-]+", key) + + +def _baggage_value(value: Any) -> str: + """``value`` as the root span shows it: a string as is, anything else as JSON (``true``, ``1``, ``["a"]``).""" + return value if isinstance(value, str) else json.dumps(value, default=str) + + @dataclass(frozen=True) class JoinedRun: """The experiment a joined conversation's gen-ai spans were told they belong to.""" diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index 79f9cb818..eafb00a71 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -1291,6 +1291,38 @@ def test_run_index_follows_the_order_conversations_are_first_sent(monkeypatch: p assert ids == [experiment_id_for(name) for name in names] +def test_the_run_metadata_rides_along_as_experiment_metadata(monkeypatch: pytest.MonkeyPatch) -> None: + """Without it, a gen-ai span that names the experiment can leave its metadata empty.""" + scope = ItemTraceScope( + base_name="ds_ts_model", + suffix_runs=True, + dataset_id="ds-1", + item_id="item-1", + run_metadata={ + "github_ref_name": "feat/a b", + "eval_k": 1, + "tags": ["x", "y"], + "rerun": True, + "unset": None, + "ci run,id": "7", + "zkouška": "x", + }, + ) + requests: list[httpx.Request] = [] + client = _scoped_client(monkeypatch, requests) + with item_scope(scope): + client.send_message("conv-metadata", "q") + + (baggage,) = _baggage_of(requests) + prefix = "langfuse_experiment_metadata_" + assert {k[len(prefix) :]: v for k, v in baggage.items() if k.startswith(prefix)} == { + "github_ref_name": "feat/a b", + "eval_k": "1", + "tags": '["x", "y"]', + "rerun": "true", + } + + def test_labels_and_scope_share_one_baggage_header(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", "github_run_id=42") requests: list[httpx.Request] = []