Skip to content

Commit bbe1187

Browse files
committed
test(gooddata-eval): add an in-process fake Langfuse server for wire-level tests
Adds tests/_fake_langfuse.py, a threading.Thread-hosted http.server that answers the five Langfuse v4 endpoints the package uses plus the three legacy ones, records every request, and is configurable per test (dataset items, missing ids, observation paging/empty-first-call, OTLP status/body, scores 429-once). It doubles as a pytest fixture (fake_langfuse in conftest.py, which points the real Langfuse env vars at it) and as a runnable script that prints and records the wire traffic for manual inspection. tests/test_fake_langfuse.py exercises the fixture itself with a real httpx.Client, no mocks. jira: trivial risk: low
1 parent 89d8a89 commit bbe1187

3 files changed

Lines changed: 447 additions & 0 deletions

File tree

Lines changed: 294 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,294 @@
1+
# (C) 2026 GoodData Corporation
2+
"""In-process fake Langfuse HTTP server: a pytest fixture and a runnable wire-watching script.
3+
4+
A `threading.Thread`-hosted `http.server` answering the Langfuse v4 endpoints the package
5+
uses, plus the three legacy ones, with canned/synthesised data recorded on `requests`.
6+
"""
7+
8+
from __future__ import annotations
9+
10+
import argparse
11+
import hashlib
12+
import itertools
13+
import json
14+
import signal
15+
import sys
16+
from datetime import datetime, timedelta, timezone
17+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
18+
from pathlib import Path
19+
from threading import Thread
20+
from urllib.parse import parse_qs, urlsplit
21+
22+
_DATASET_ITEMS_PATH = "/api/public/dataset-items"
23+
_OBSERVATIONS_PATH = "/api/public/v2/observations"
24+
_OTLP_PATH = "/api/public/otel/v1/traces"
25+
_SCORES_PATH = "/api/public/scores"
26+
_TRACES_PATH = "/api/public/traces"
27+
_INGESTION_PATH = "/api/public/ingestion"
28+
_DATASET_RUN_ITEMS_PATH = "/api/public/dataset-run-items"
29+
30+
_ROOT_LATENCY_SECONDS = 12.5
31+
_CHILD_COSTS = (0.01, 0.02)
32+
33+
34+
def _short_hash(value: str, length: int) -> str:
35+
return hashlib.md5(value.encode()).hexdigest()[:length]
36+
37+
38+
def observation_rows(session_id: str) -> list[dict]:
39+
"""A gen-ai-shaped root row plus two child rows for one conversation session."""
40+
trace_id = _short_hash(session_id, 32)
41+
root_id = _short_hash(f"{session_id}:root", 16)
42+
start = datetime(2026, 1, 1, tzinfo=timezone.utc)
43+
times = {"startTime": start.isoformat(), "endTime": (start + timedelta(seconds=_ROOT_LATENCY_SECONDS)).isoformat()}
44+
shared = {"traceId": trace_id, "type": "GENERATION", "sessionId": session_id, **times}
45+
root = {
46+
"id": root_id,
47+
"name": "conversation.send_message",
48+
"parentObservationId": None,
49+
"isRootObservation": True,
50+
"latency": _ROOT_LATENCY_SECONDS,
51+
"totalCost": None,
52+
"metadata": {"conversation_id": session_id},
53+
**shared,
54+
}
55+
children = [
56+
{
57+
"id": _short_hash(f"{session_id}:child{i}", 16),
58+
"name": f"model.call.{i}",
59+
"parentObservationId": root_id,
60+
"isRootObservation": False,
61+
"latency": None,
62+
"totalCost": cost,
63+
"metadata": {},
64+
**shared,
65+
}
66+
for i, cost in enumerate(_CHILD_COSTS, start=1)
67+
]
68+
return [root, *children]
69+
70+
71+
def _canned_item(item_id: str, dataset_name: str) -> dict:
72+
return {
73+
"id": item_id,
74+
"datasetName": dataset_name,
75+
"input": {"question": f"Canned question for {item_id}"},
76+
"expectedOutput": f"Rubric: the answer for {item_id} must directly address the question.",
77+
"metadata": {},
78+
}
79+
80+
81+
class FakeLangfuse:
82+
"""A fake Langfuse server on `127.0.0.1:0`, configurable per test before each request."""
83+
84+
def __init__(self, *, port: int = 0, verbose: bool = False) -> None:
85+
self.dataset_name = "fake"
86+
self.dataset_id = "ds-fake"
87+
self.items: list[dict] = []
88+
self.missing: set[str] = set()
89+
self.first_observations_call_empty = False
90+
self.observations_pages = 1
91+
self.otlp_status = 200
92+
self.otlp_body: dict = {}
93+
self.scores_429_once = False
94+
self.ingestion_body: dict = {"successes": [], "errors": []}
95+
self.verbose = verbose
96+
self.requests: list[dict] = []
97+
self.on_request = None
98+
handler = self._make_handler()
99+
self._httpd = ThreadingHTTPServer(("127.0.0.1", port), handler)
100+
self._thread = Thread(target=self._httpd.serve_forever, daemon=True)
101+
102+
@property
103+
def base_url(self) -> str:
104+
host, port = self._httpd.server_address[:2]
105+
return f"http://{host}:{port}"
106+
107+
def start(self) -> None:
108+
self._thread.start()
109+
110+
def stop(self) -> None:
111+
self._httpd.shutdown()
112+
self._httpd.server_close()
113+
114+
def __enter__(self) -> FakeLangfuse:
115+
self.start()
116+
return self
117+
118+
def __exit__(self, *exc_info: object) -> None:
119+
self.stop()
120+
121+
def calls(self, method: str, path_prefix: str) -> list[dict]:
122+
return [r for r in self.requests if r["method"] == method and r["path"].startswith(path_prefix)]
123+
124+
def _make_handler(self) -> type[BaseHTTPRequestHandler]:
125+
server = self
126+
127+
class Handler(BaseHTTPRequestHandler):
128+
def log_message(self, format: str, *args: object) -> None:
129+
if server.verbose:
130+
super().log_message(format, *args)
131+
132+
def do_GET(self) -> None:
133+
server._handle(self, "GET")
134+
135+
def do_POST(self) -> None:
136+
server._handle(self, "POST")
137+
138+
return Handler
139+
140+
def _handle(self, handler: BaseHTTPRequestHandler, method: str) -> None:
141+
try:
142+
self._route(handler, method)
143+
except Exception as exc: # never let a bad request kill the server thread
144+
self._respond(handler, 400, {"message": f"fake_langfuse error: {exc}"})
145+
146+
def _route(self, handler: BaseHTTPRequestHandler, method: str) -> None:
147+
parts = urlsplit(handler.path)
148+
path = parts.path
149+
query = {k: v[0] for k, v in parse_qs(parts.query).items()}
150+
body = self._read_json_body(handler)
151+
record = {
152+
"method": method,
153+
"path": path,
154+
"query": query,
155+
"headers": {k.lower(): v for k, v in handler.headers.items()},
156+
"json": body,
157+
}
158+
self.requests.append(record)
159+
if self.on_request is not None:
160+
self.on_request(record)
161+
162+
if method == "GET" and path == _DATASET_ITEMS_PATH:
163+
self._get_dataset_items(handler, query)
164+
elif method == "GET" and path.startswith(f"{_DATASET_ITEMS_PATH}/"):
165+
self._get_dataset_item(handler, path.rsplit("/", 1)[-1])
166+
elif method == "GET" and path == _OBSERVATIONS_PATH:
167+
self._get_observations(handler, query)
168+
elif method == "POST" and path == _OTLP_PATH:
169+
self._respond(handler, self.otlp_status, self.otlp_body)
170+
elif method == "POST" and path == _SCORES_PATH:
171+
self._post_scores(handler)
172+
elif method == "GET" and path == _TRACES_PATH:
173+
self._get_traces(handler, query)
174+
elif method == "POST" and path == _INGESTION_PATH:
175+
self._respond(handler, 200, self.ingestion_body)
176+
elif method == "POST" and path == _DATASET_RUN_ITEMS_PATH:
177+
self._respond(handler, 200, {})
178+
else:
179+
self._respond(handler, 404, {"message": f"fake_langfuse: no route for {method} {path}"})
180+
181+
@staticmethod
182+
def _read_json_body(handler: BaseHTTPRequestHandler) -> dict | None:
183+
length = int(handler.headers.get("Content-Length") or 0)
184+
if not length:
185+
return None
186+
raw = handler.rfile.read(length)
187+
return json.loads(raw) if raw else None
188+
189+
def _get_dataset_items(self, handler: BaseHTTPRequestHandler, query: dict) -> None:
190+
if query.get("datasetName") != self.dataset_name:
191+
self._respond(handler, 404, {"message": "Dataset not found"})
192+
return
193+
meta = {"page": 1, "limit": 100, "totalItems": len(self.items), "totalPages": 1}
194+
self._respond(handler, 200, {"data": self.items, "meta": meta})
195+
196+
def _get_dataset_item(self, handler: BaseHTTPRequestHandler, item_id: str) -> None:
197+
if item_id in self.missing:
198+
self._respond(handler, 404, {"message": "Dataset item not found"})
199+
return
200+
self._respond(handler, 200, {"id": item_id, "datasetId": self.dataset_id, "datasetName": self.dataset_name})
201+
202+
def _post_scores(self, handler: BaseHTTPRequestHandler) -> None:
203+
if self.scores_429_once:
204+
self.scores_429_once = False
205+
self._respond(handler, 429, {"message": "rate limited"}, headers={"Retry-After": "0"})
206+
return
207+
self._respond(handler, 200, {})
208+
209+
def _get_observations(self, handler: BaseHTTPRequestHandler, query: dict) -> None:
210+
session_id = query.get("sessionId", "")
211+
if self.first_observations_call_empty:
212+
self.first_observations_call_empty = False
213+
self._respond(handler, 200, {"data": [], "meta": {}})
214+
return
215+
rows = observation_rows(session_id)
216+
if self.observations_pages == 2 and query.get("cursor") is None:
217+
self._respond(handler, 200, {"data": rows[:1], "meta": {"cursor": "page-2"}})
218+
return
219+
if self.observations_pages == 2:
220+
self._respond(handler, 200, {"data": rows[1:], "meta": {}})
221+
return
222+
self._respond(handler, 200, {"data": rows, "meta": {}})
223+
224+
def _get_traces(self, handler: BaseHTTPRequestHandler, query: dict) -> None:
225+
session_id = query.get("sessionId", "")
226+
legacy_trace = {
227+
"id": _short_hash(session_id, 32),
228+
"sessionId": session_id,
229+
"latency": _ROOT_LATENCY_SECONDS,
230+
"totalCost": sum(_CHILD_COSTS),
231+
"metadata": {"conversation_id": session_id},
232+
}
233+
self._respond(handler, 200, {"data": [legacy_trace]})
234+
235+
def _respond(
236+
self, handler: BaseHTTPRequestHandler, status: int, body: dict, *, headers: dict | None = None
237+
) -> None:
238+
payload = json.dumps(body).encode()
239+
handler.send_response(status)
240+
handler.send_header("Content-Type", "application/json")
241+
handler.send_header("Content-Length", str(len(payload)))
242+
for key, value in (headers or {}).items():
243+
handler.send_header(key, value)
244+
handler.end_headers()
245+
handler.wfile.write(payload)
246+
247+
248+
def _parse_args(argv: list[str]) -> argparse.Namespace:
249+
parser = argparse.ArgumentParser(description="Run a fake Langfuse server and watch the wire.")
250+
parser.add_argument("--port", type=int, default=8765)
251+
parser.add_argument("--dataset", default="fake")
252+
parser.add_argument("--items", nargs="*", default=[])
253+
parser.add_argument("--record", type=Path, default=None, help="Directory to write each request body to.")
254+
parser.add_argument("--verbose", action="store_true")
255+
return parser.parse_args(argv)
256+
257+
258+
def _make_recorder(record_dir: Path):
259+
record_dir.mkdir(parents=True, exist_ok=True)
260+
numbers = itertools.count(1)
261+
262+
def _record(record: dict) -> None:
263+
safe_path = record["path"].replace("/", "_")
264+
out = record_dir / f"{next(numbers):03d}-{record['method']}-{safe_path}.json"
265+
out.write_text(json.dumps(record.get("json"), indent=2, default=str))
266+
267+
return _record
268+
269+
270+
def main(argv: list[str]) -> int:
271+
args = _parse_args(argv)
272+
server = FakeLangfuse(port=args.port, verbose=args.verbose)
273+
server.dataset_name = args.dataset
274+
server.items = [_canned_item(item_id, args.dataset) for item_id in args.items]
275+
276+
recorder = _make_recorder(args.record) if args.record else None
277+
278+
def _on_request(record: dict) -> None:
279+
print(json.dumps(record, indent=2, default=str))
280+
if recorder is not None:
281+
recorder(record)
282+
283+
server.on_request = _on_request
284+
with server:
285+
print(f"fake Langfuse server listening on {server.base_url}", file=sys.stderr)
286+
try:
287+
signal.pause()
288+
except KeyboardInterrupt:
289+
pass
290+
return 0
291+
292+
293+
if __name__ == "__main__":
294+
raise SystemExit(main(sys.argv[1:]))

‎packages/gooddata-eval/tests/conftest.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,20 @@
33

44
import pytest
55

6+
from tests._fake_langfuse import FakeLangfuse
7+
68

79
@pytest.fixture
810
def fixtures_dir() -> Path:
911
return Path(__file__).parent / "fixtures"
12+
13+
14+
@pytest.fixture
15+
def fake_langfuse(monkeypatch: pytest.MonkeyPatch):
16+
"""A running fake Langfuse server with the real client's env vars pointed at it."""
17+
with FakeLangfuse() as server:
18+
monkeypatch.setenv("LANGFUSE_BASE_URL", server.base_url)
19+
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-fake")
20+
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-fake")
21+
monkeypatch.delenv("LANGFUSE_HOST", raising=False)
22+
yield server

0 commit comments

Comments
 (0)