|
| 1 | +# (C) 2026 GoodData Corporation |
| 2 | +"""In-process fake Langfuse HTTP server: a pytest fixture and a runnable wire-watching script. |
| 3 | +
|
| 4 | +A `threading.Thread`-hosted `http.server` answering the Langfuse v4 endpoints the package |
| 5 | +uses, plus the three legacy ones, with canned/synthesised data recorded on `requests`. |
| 6 | +""" |
| 7 | + |
| 8 | +from __future__ import annotations |
| 9 | + |
| 10 | +import argparse |
| 11 | +import hashlib |
| 12 | +import itertools |
| 13 | +import json |
| 14 | +import signal |
| 15 | +import sys |
| 16 | +from datetime import datetime, timedelta, timezone |
| 17 | +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer |
| 18 | +from pathlib import Path |
| 19 | +from threading import Thread |
| 20 | +from urllib.parse import parse_qs, urlsplit |
| 21 | + |
| 22 | +_DATASET_ITEMS_PATH = "/api/public/dataset-items" |
| 23 | +_OBSERVATIONS_PATH = "/api/public/v2/observations" |
| 24 | +_OTLP_PATH = "/api/public/otel/v1/traces" |
| 25 | +_SCORES_PATH = "/api/public/scores" |
| 26 | +_TRACES_PATH = "/api/public/traces" |
| 27 | +_INGESTION_PATH = "/api/public/ingestion" |
| 28 | +_DATASET_RUN_ITEMS_PATH = "/api/public/dataset-run-items" |
| 29 | + |
| 30 | +_ROOT_LATENCY_SECONDS = 12.5 |
| 31 | +_CHILD_COSTS = (0.01, 0.02) |
| 32 | + |
| 33 | + |
| 34 | +def _short_hash(value: str, length: int) -> str: |
| 35 | + return hashlib.md5(value.encode()).hexdigest()[:length] |
| 36 | + |
| 37 | + |
| 38 | +def observation_rows(session_id: str) -> list[dict]: |
| 39 | + """A gen-ai-shaped root row plus two child rows for one conversation session.""" |
| 40 | + trace_id = _short_hash(session_id, 32) |
| 41 | + root_id = _short_hash(f"{session_id}:root", 16) |
| 42 | + start = datetime(2026, 1, 1, tzinfo=timezone.utc) |
| 43 | + times = {"startTime": start.isoformat(), "endTime": (start + timedelta(seconds=_ROOT_LATENCY_SECONDS)).isoformat()} |
| 44 | + shared = {"traceId": trace_id, "type": "GENERATION", "sessionId": session_id, **times} |
| 45 | + root = { |
| 46 | + "id": root_id, |
| 47 | + "name": "conversation.send_message", |
| 48 | + "parentObservationId": None, |
| 49 | + "isRootObservation": True, |
| 50 | + "latency": _ROOT_LATENCY_SECONDS, |
| 51 | + "totalCost": None, |
| 52 | + "metadata": {"conversation_id": session_id}, |
| 53 | + **shared, |
| 54 | + } |
| 55 | + children = [ |
| 56 | + { |
| 57 | + "id": _short_hash(f"{session_id}:child{i}", 16), |
| 58 | + "name": f"model.call.{i}", |
| 59 | + "parentObservationId": root_id, |
| 60 | + "isRootObservation": False, |
| 61 | + "latency": None, |
| 62 | + "totalCost": cost, |
| 63 | + "metadata": {}, |
| 64 | + **shared, |
| 65 | + } |
| 66 | + for i, cost in enumerate(_CHILD_COSTS, start=1) |
| 67 | + ] |
| 68 | + return [root, *children] |
| 69 | + |
| 70 | + |
| 71 | +def _canned_item(item_id: str, dataset_name: str) -> dict: |
| 72 | + return { |
| 73 | + "id": item_id, |
| 74 | + "datasetName": dataset_name, |
| 75 | + "input": {"question": f"Canned question for {item_id}"}, |
| 76 | + "expectedOutput": f"Rubric: the answer for {item_id} must directly address the question.", |
| 77 | + "metadata": {}, |
| 78 | + } |
| 79 | + |
| 80 | + |
| 81 | +class FakeLangfuse: |
| 82 | + """A fake Langfuse server on `127.0.0.1:0`, configurable per test before each request.""" |
| 83 | + |
| 84 | + def __init__(self, *, port: int = 0, verbose: bool = False) -> None: |
| 85 | + self.dataset_name = "fake" |
| 86 | + self.dataset_id = "ds-fake" |
| 87 | + self.items: list[dict] = [] |
| 88 | + self.missing: set[str] = set() |
| 89 | + self.first_observations_call_empty = False |
| 90 | + self.observations_pages = 1 |
| 91 | + self.otlp_status = 200 |
| 92 | + self.otlp_body: dict = {} |
| 93 | + self.scores_429_once = False |
| 94 | + self.ingestion_body: dict = {"successes": [], "errors": []} |
| 95 | + self.verbose = verbose |
| 96 | + self.requests: list[dict] = [] |
| 97 | + self.on_request = None |
| 98 | + handler = self._make_handler() |
| 99 | + self._httpd = ThreadingHTTPServer(("127.0.0.1", port), handler) |
| 100 | + self._thread = Thread(target=self._httpd.serve_forever, daemon=True) |
| 101 | + |
| 102 | + @property |
| 103 | + def base_url(self) -> str: |
| 104 | + host, port = self._httpd.server_address[:2] |
| 105 | + return f"http://{host}:{port}" |
| 106 | + |
| 107 | + def start(self) -> None: |
| 108 | + self._thread.start() |
| 109 | + |
| 110 | + def stop(self) -> None: |
| 111 | + self._httpd.shutdown() |
| 112 | + self._httpd.server_close() |
| 113 | + |
| 114 | + def __enter__(self) -> FakeLangfuse: |
| 115 | + self.start() |
| 116 | + return self |
| 117 | + |
| 118 | + def __exit__(self, *exc_info: object) -> None: |
| 119 | + self.stop() |
| 120 | + |
| 121 | + def calls(self, method: str, path_prefix: str) -> list[dict]: |
| 122 | + return [r for r in self.requests if r["method"] == method and r["path"].startswith(path_prefix)] |
| 123 | + |
| 124 | + def _make_handler(self) -> type[BaseHTTPRequestHandler]: |
| 125 | + server = self |
| 126 | + |
| 127 | + class Handler(BaseHTTPRequestHandler): |
| 128 | + def log_message(self, format: str, *args: object) -> None: |
| 129 | + if server.verbose: |
| 130 | + super().log_message(format, *args) |
| 131 | + |
| 132 | + def do_GET(self) -> None: |
| 133 | + server._handle(self, "GET") |
| 134 | + |
| 135 | + def do_POST(self) -> None: |
| 136 | + server._handle(self, "POST") |
| 137 | + |
| 138 | + return Handler |
| 139 | + |
| 140 | + def _handle(self, handler: BaseHTTPRequestHandler, method: str) -> None: |
| 141 | + try: |
| 142 | + self._route(handler, method) |
| 143 | + except Exception as exc: # never let a bad request kill the server thread |
| 144 | + self._respond(handler, 400, {"message": f"fake_langfuse error: {exc}"}) |
| 145 | + |
| 146 | + def _route(self, handler: BaseHTTPRequestHandler, method: str) -> None: |
| 147 | + parts = urlsplit(handler.path) |
| 148 | + path = parts.path |
| 149 | + query = {k: v[0] for k, v in parse_qs(parts.query).items()} |
| 150 | + body = self._read_json_body(handler) |
| 151 | + record = { |
| 152 | + "method": method, |
| 153 | + "path": path, |
| 154 | + "query": query, |
| 155 | + "headers": {k.lower(): v for k, v in handler.headers.items()}, |
| 156 | + "json": body, |
| 157 | + } |
| 158 | + self.requests.append(record) |
| 159 | + if self.on_request is not None: |
| 160 | + self.on_request(record) |
| 161 | + |
| 162 | + if method == "GET" and path == _DATASET_ITEMS_PATH: |
| 163 | + self._get_dataset_items(handler, query) |
| 164 | + elif method == "GET" and path.startswith(f"{_DATASET_ITEMS_PATH}/"): |
| 165 | + self._get_dataset_item(handler, path.rsplit("/", 1)[-1]) |
| 166 | + elif method == "GET" and path == _OBSERVATIONS_PATH: |
| 167 | + self._get_observations(handler, query) |
| 168 | + elif method == "POST" and path == _OTLP_PATH: |
| 169 | + self._respond(handler, self.otlp_status, self.otlp_body) |
| 170 | + elif method == "POST" and path == _SCORES_PATH: |
| 171 | + self._post_scores(handler) |
| 172 | + elif method == "GET" and path == _TRACES_PATH: |
| 173 | + self._get_traces(handler, query) |
| 174 | + elif method == "POST" and path == _INGESTION_PATH: |
| 175 | + self._respond(handler, 200, self.ingestion_body) |
| 176 | + elif method == "POST" and path == _DATASET_RUN_ITEMS_PATH: |
| 177 | + self._respond(handler, 200, {}) |
| 178 | + else: |
| 179 | + self._respond(handler, 404, {"message": f"fake_langfuse: no route for {method} {path}"}) |
| 180 | + |
| 181 | + @staticmethod |
| 182 | + def _read_json_body(handler: BaseHTTPRequestHandler) -> dict | None: |
| 183 | + length = int(handler.headers.get("Content-Length") or 0) |
| 184 | + if not length: |
| 185 | + return None |
| 186 | + raw = handler.rfile.read(length) |
| 187 | + return json.loads(raw) if raw else None |
| 188 | + |
| 189 | + def _get_dataset_items(self, handler: BaseHTTPRequestHandler, query: dict) -> None: |
| 190 | + if query.get("datasetName") != self.dataset_name: |
| 191 | + self._respond(handler, 404, {"message": "Dataset not found"}) |
| 192 | + return |
| 193 | + meta = {"page": 1, "limit": 100, "totalItems": len(self.items), "totalPages": 1} |
| 194 | + self._respond(handler, 200, {"data": self.items, "meta": meta}) |
| 195 | + |
| 196 | + def _get_dataset_item(self, handler: BaseHTTPRequestHandler, item_id: str) -> None: |
| 197 | + if item_id in self.missing: |
| 198 | + self._respond(handler, 404, {"message": "Dataset item not found"}) |
| 199 | + return |
| 200 | + self._respond(handler, 200, {"id": item_id, "datasetId": self.dataset_id, "datasetName": self.dataset_name}) |
| 201 | + |
| 202 | + def _post_scores(self, handler: BaseHTTPRequestHandler) -> None: |
| 203 | + if self.scores_429_once: |
| 204 | + self.scores_429_once = False |
| 205 | + self._respond(handler, 429, {"message": "rate limited"}, headers={"Retry-After": "0"}) |
| 206 | + return |
| 207 | + self._respond(handler, 200, {}) |
| 208 | + |
| 209 | + def _get_observations(self, handler: BaseHTTPRequestHandler, query: dict) -> None: |
| 210 | + session_id = query.get("sessionId", "") |
| 211 | + if self.first_observations_call_empty: |
| 212 | + self.first_observations_call_empty = False |
| 213 | + self._respond(handler, 200, {"data": [], "meta": {}}) |
| 214 | + return |
| 215 | + rows = observation_rows(session_id) |
| 216 | + if self.observations_pages == 2 and query.get("cursor") is None: |
| 217 | + self._respond(handler, 200, {"data": rows[:1], "meta": {"cursor": "page-2"}}) |
| 218 | + return |
| 219 | + if self.observations_pages == 2: |
| 220 | + self._respond(handler, 200, {"data": rows[1:], "meta": {}}) |
| 221 | + return |
| 222 | + self._respond(handler, 200, {"data": rows, "meta": {}}) |
| 223 | + |
| 224 | + def _get_traces(self, handler: BaseHTTPRequestHandler, query: dict) -> None: |
| 225 | + session_id = query.get("sessionId", "") |
| 226 | + legacy_trace = { |
| 227 | + "id": _short_hash(session_id, 32), |
| 228 | + "sessionId": session_id, |
| 229 | + "latency": _ROOT_LATENCY_SECONDS, |
| 230 | + "totalCost": sum(_CHILD_COSTS), |
| 231 | + "metadata": {"conversation_id": session_id}, |
| 232 | + } |
| 233 | + self._respond(handler, 200, {"data": [legacy_trace]}) |
| 234 | + |
| 235 | + def _respond( |
| 236 | + self, handler: BaseHTTPRequestHandler, status: int, body: dict, *, headers: dict | None = None |
| 237 | + ) -> None: |
| 238 | + payload = json.dumps(body).encode() |
| 239 | + handler.send_response(status) |
| 240 | + handler.send_header("Content-Type", "application/json") |
| 241 | + handler.send_header("Content-Length", str(len(payload))) |
| 242 | + for key, value in (headers or {}).items(): |
| 243 | + handler.send_header(key, value) |
| 244 | + handler.end_headers() |
| 245 | + handler.wfile.write(payload) |
| 246 | + |
| 247 | + |
| 248 | +def _parse_args(argv: list[str]) -> argparse.Namespace: |
| 249 | + parser = argparse.ArgumentParser(description="Run a fake Langfuse server and watch the wire.") |
| 250 | + parser.add_argument("--port", type=int, default=8765) |
| 251 | + parser.add_argument("--dataset", default="fake") |
| 252 | + parser.add_argument("--items", nargs="*", default=[]) |
| 253 | + parser.add_argument("--record", type=Path, default=None, help="Directory to write each request body to.") |
| 254 | + parser.add_argument("--verbose", action="store_true") |
| 255 | + return parser.parse_args(argv) |
| 256 | + |
| 257 | + |
| 258 | +def _make_recorder(record_dir: Path): |
| 259 | + record_dir.mkdir(parents=True, exist_ok=True) |
| 260 | + numbers = itertools.count(1) |
| 261 | + |
| 262 | + def _record(record: dict) -> None: |
| 263 | + safe_path = record["path"].replace("/", "_") |
| 264 | + out = record_dir / f"{next(numbers):03d}-{record['method']}-{safe_path}.json" |
| 265 | + out.write_text(json.dumps(record.get("json"), indent=2, default=str)) |
| 266 | + |
| 267 | + return _record |
| 268 | + |
| 269 | + |
| 270 | +def main(argv: list[str]) -> int: |
| 271 | + args = _parse_args(argv) |
| 272 | + server = FakeLangfuse(port=args.port, verbose=args.verbose) |
| 273 | + server.dataset_name = args.dataset |
| 274 | + server.items = [_canned_item(item_id, args.dataset) for item_id in args.items] |
| 275 | + |
| 276 | + recorder = _make_recorder(args.record) if args.record else None |
| 277 | + |
| 278 | + def _on_request(record: dict) -> None: |
| 279 | + print(json.dumps(record, indent=2, default=str)) |
| 280 | + if recorder is not None: |
| 281 | + recorder(record) |
| 282 | + |
| 283 | + server.on_request = _on_request |
| 284 | + with server: |
| 285 | + print(f"fake Langfuse server listening on {server.base_url}", file=sys.stderr) |
| 286 | + try: |
| 287 | + signal.pause() |
| 288 | + except KeyboardInterrupt: |
| 289 | + pass |
| 290 | + return 0 |
| 291 | + |
| 292 | + |
| 293 | +if __name__ == "__main__": |
| 294 | + raise SystemExit(main(sys.argv[1:])) |
0 commit comments