Skip to content

Commit 1ae46c4

Browse files
committed
feat(gooddata-eval): label gen-ai traces through W3C baggage
Langfuse v4 cannot add a label to an observation after it is created: the trace-create ingestion event stops working on 2026-11-16 and no endpoint updates an existing observation. A caller that wants its gen-ai traces grouped by combo or CI run therefore has to send the label with the request. ChatClient now reads GOODDATA_EVAL_TRACE_LABELS (key=value pairs, comma-separated) and sends them as a W3C baggage header on every chat request: each label as langfuse_metadata_<key>, and model_version also as langfuse_version, the dimension Langfuse groups cost by. gen-ai's Langfuse span processor copies langfuse_* baggage onto every span it starts, so the root generation, the cost-bearing children, retries and the title trace all carry the labels. Values are percent-encoded; OTel decodes them with unquote_plus, so '+' and spaces survive. Unset or empty sends no header, which keeps every existing run unchanged. Verified against gen-ai's own langfuse 4.14.4 SDK and FastAPI instrumentation: the header this builds lands as langfuse.version and langfuse.trace.metadata.* on both the root and the child generation. gen-ai only keeps caller langfuse_* baggage where genAi.langfuse.acceptCallerBaggage is on (gooddata/gdc-nas#27251). jira: trivial risk: low
1 parent 0a7b141 commit 1ae46c4

3 files changed

Lines changed: 89 additions & 1 deletion

File tree

‎packages/gooddata-eval/AGENTS.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@ Run this before guessing a `--model` string.
156156
| `GD_EVAL_AGENT_ID` | which agent to drive, same as `--agent-id` |
157157
| `LANGFUSE_HOST`, `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` | `--langfuse`, `--langfuse-dataset` |
158158
| `GOODDATA_EVAL_CHAT_*` | SSE retry, backoff and timeout knobs |
159+
| `GOODDATA_EVAL_TRACE_LABELS` | `key=value,...` labels stamped on every observation of a chat conversation via W3C baggage; `model_version` also sets the observation `version`. Dashboard-summary runs are not labelled |
159160
| `GD_EVAL_TIMERS` | same as `--timers` |
160161

161162
A gitignored `.env` at the repo root is the normal place for these; load it with

‎packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import time
2020
from dataclasses import dataclass, field
2121
from typing import Any, Callable, Iterable, TypeVar
22+
from urllib.parse import quote
2223

2324
import httpx
2425

@@ -135,6 +136,32 @@ def _float_env(name: str, default: float) -> float:
135136
# 4-turn conversation run to 4x the budget, which is not what a user would sit through.
136137
_ITEM_TIMEOUT_S = _float_env("GOODDATA_EVAL_CHAT_ITEM_TIMEOUT_S", 0.0)
137138

139+
# Comma-separated `key=value` labels, e.g. `model_version=gpt-5.2,github_run_id=42`, sent as
140+
# W3C baggage on every chat request. gen-ai's Langfuse span processor copies `langfuse_*`
141+
# baggage onto each span it starts, so every observation of the conversation -- the cost-bearing
142+
# generations, retries and the title trace included -- carries them from creation. Langfuse v4
143+
# cannot add them to an observation afterwards. A value cannot contain a comma, and a key must be
144+
# a plain token (letters, digits, `_`, `-`): the receiver drops any other key.
145+
_TRACE_LABELS_ENV = "GOODDATA_EVAL_TRACE_LABELS"
146+
147+
148+
def _trace_baggage() -> dict[str, str]:
149+
"""The `baggage` header for the labels in ``GOODDATA_EVAL_TRACE_LABELS``, or none.
150+
151+
Each label becomes trace metadata, and ``model_version`` -- the combo the report groups
152+
on -- also becomes the observation ``version``, the dimension Langfuse groups cost by.
153+
Values are percent-encoded: a space or comma would otherwise end the baggage entry.
154+
"""
155+
labels: dict[str, str] = {}
156+
for pair in os.getenv(_TRACE_LABELS_ENV, "").split(","):
157+
key, sep, value = pair.partition("=")
158+
if sep and key.strip():
159+
labels[key.strip()] = value.strip()
160+
entries = [f"langfuse_metadata_{k}={quote(v, safe='')}" for k, v in labels.items()]
161+
if labels.get("model_version"):
162+
entries.insert(0, f"langfuse_version={quote(labels['model_version'], safe='')}")
163+
return {"baggage": ",".join(entries)} if entries else {}
164+
138165

139166
def set_default_turn_timeout(seconds: float | None) -> None:
140167
"""Set the per-turn budget every ChatClient built afterwards inherits.
@@ -466,7 +493,7 @@ def __init__(
466493
the ``enableAiContextSetup`` feature flag is on for the organization.
467494
"""
468495
self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations"
469-
self._auth = {"Authorization": f"Bearer {token}"}
496+
self._auth = {"Authorization": f"Bearer {token}", **_trace_baggage()}
470497
# 0/None disables the cap. Also lowered onto the read timeout: the wall-clock check
471498
# fires between events, so a turn that goes silent needs the transport to give up too.
472499
budget = _TURN_TIMEOUT_S if turn_timeout_s is None else turn_timeout_s

‎packages/gooddata-eval/tests/test_sse_client.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
# (C) 2026 GoodData Corporation
22
import json
3+
from collections.abc import Callable
4+
from urllib.parse import unquote
35

46
import httpx
57
import pytest
@@ -1053,3 +1055,61 @@ def handler(request):
10531055
)
10541056
client.ask(item)
10551057
assert captured["body"]["userContext"] == _ATTACHMENT
1058+
1059+
1060+
def _baggage_of(requests: list[httpx.Request]) -> list[dict[str, str] | None]:
1061+
"""Each request's `baggage` header as {key: decoded value}, or None when absent."""
1062+
out: list[dict[str, str] | None] = []
1063+
for r in requests:
1064+
raw = r.headers.get("baggage")
1065+
out.append(None if raw is None else {k: unquote(v) for k, v in (e.split("=", 1) for e in raw.split(","))})
1066+
return out
1067+
1068+
1069+
def _record_requests(requests: list[httpx.Request]) -> Callable[[httpx.Request], httpx.Response]:
1070+
def handler(request: httpx.Request) -> httpx.Response:
1071+
requests.append(request)
1072+
if request.url.path.endswith("/conversations"):
1073+
return httpx.Response(200, json={"conversationId": "c1"})
1074+
return httpx.Response(200, content=_OK_SSE)
1075+
1076+
return handler
1077+
1078+
1079+
def test_trace_labels_ride_every_request_as_langfuse_baggage(monkeypatch: pytest.MonkeyPatch) -> None:
1080+
"""gen-ai's Langfuse span processor copies `langfuse_*` baggage onto every span it starts, so
1081+
the labels reach the root generation and its cost-bearing children at creation."""
1082+
monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", "model_version=gpt-5.2 openai,github_run_id=42")
1083+
requests: list[httpx.Request] = []
1084+
client = _client_with_handler(_record_requests(requests))
1085+
1086+
client.send_message(client.create_conversation(), "q")
1087+
1088+
expected = {
1089+
"langfuse_version": "gpt-5.2 openai",
1090+
"langfuse_metadata_model_version": "gpt-5.2 openai",
1091+
"langfuse_metadata_github_run_id": "42",
1092+
}
1093+
assert _baggage_of(requests) == [expected, expected]
1094+
# Encoded, not raw: a space or comma in a value would otherwise split or end the entry.
1095+
assert " " not in requests[0].headers["baggage"]
1096+
1097+
1098+
def test_trace_labels_without_a_model_version_set_no_langfuse_version(monkeypatch: pytest.MonkeyPatch) -> None:
1099+
monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", "github_run_id=42")
1100+
requests: list[httpx.Request] = []
1101+
_client_with_handler(_record_requests(requests)).create_conversation()
1102+
1103+
assert _baggage_of(requests) == [{"langfuse_metadata_github_run_id": "42"}]
1104+
1105+
1106+
@pytest.mark.parametrize("raw", [None, "", "no-equals-sign", " , "])
1107+
def test_no_trace_labels_send_no_baggage(monkeypatch: pytest.MonkeyPatch, raw: str | None) -> None:
1108+
if raw is None:
1109+
monkeypatch.delenv("GOODDATA_EVAL_TRACE_LABELS", raising=False)
1110+
else:
1111+
monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", raw)
1112+
requests: list[httpx.Request] = []
1113+
_client_with_handler(_record_requests(requests)).create_conversation()
1114+
1115+
assert _baggage_of(requests) == [None]

0 commit comments

Comments
 (0)