feat(evaluations): pass the tool trajectory to scorers

packages/client/src/launchdarkly_ai_server/evaluations/__init__.py

@@ -9,7 +9,7 @@
     Transport,
     urllib_transport,
 )
-from .criteria import Criterion, Judge, Scorer, SuccessDirection
+from .criteria import Criterion, Judge, Scorer, ScorerContext, SuccessDirection
 from .module import EvaluationsModule, init_evaluations
 from .types import (
     AIConfig,
@@ -35,6 +35,7 @@
     "LDApiError",
     "RunSummary",
     "Scorer",
+    "ScorerContext",
     "SuccessDirection",
     "Transport",
     "Usage",

packages/client/src/launchdarkly_ai_server/evaluations/criteria.py

@@ -1,13 +1,19 @@
 from __future__ import annotations
 
+import inspect
 from collections.abc import Awaitable, Callable
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from math import isnan
 from typing import Any, Literal
 
+from ..trajectory import ToolInvocation
 from .types import DatasetRow
 
-type ScorerFn = Callable[[DatasetRow, Any], float | bool | Awaitable[float | bool]]
+type ScorerResult = float | bool | Awaitable[float | bool]
+type ScorerFn = (
+    Callable[[DatasetRow, Any], ScorerResult]
+    | Callable[[DatasetRow, Any, ScorerContext], ScorerResult]
+)
 
 type SuccessDirection = Literal["higher_is_better", "lower_is_better"]
 
@@ -63,15 +69,36 @@ def to_criteria_wire(self) -> dict[str, Any]:
         }
 
 
+@dataclass(frozen=True)
+class ScorerContext:
+    """Extra inputs for a scorer, beyond the row and the generated output.
+
+    A scorer receives this as an optional third argument. New inputs are added
+    as new fields, so the scorer signature does not change.
+
+    ``tool_calls`` lists the tools the handler called while it produced the
+    output, in the order the calls started. It is empty when the handler
+    called no tool. At most 50 calls are recorded. ``tool_calls_omitted``
+    counts the calls made past that limit. Each call records its arguments and
+    result as text, cut to 2000 characters.
+    """
+
+    tool_calls: tuple[ToolInvocation, ...] = ()
+    tool_calls_omitted: int = 0
+
+
 @dataclass(frozen=True)
 class Scorer:
     """Local deterministic scorer run for each generated evaluation row.
 
     ``fn`` may be sync or async and receives ``(row, output)``, where ``row``
     is the :class:`~launchdarkly_ai_server.evaluations.types.DatasetRow` the
-    output was generated from and ``output`` is the generated output. It must
-    return a boolean or a numeric score from 0 to 1. Boolean results are
-    converted to 1.0 or 0.0 before being emitted as evaluation events.
+    output was generated from and ``output`` is the generated output. To
+    receive more inputs, declare a third positional parameter. It receives a
+    :class:`ScorerContext`. A function that declares only two parameters is
+    called with two arguments. ``fn`` must return a boolean or a numeric score
+    from 0 to 1. Boolean results are converted to 1.0 or 0.0 before being
+    emitted as evaluation events.
 
     ``threshold`` defaults to 1.0: a row passes only on a perfect score, which
     matches the common case of boolean scorers. Pass a lower threshold for
@@ -90,12 +117,14 @@ class Scorer:
     threshold: float | None = 1.0
     pass_rate_threshold: float | None = None
     success_direction: SuccessDirection = "higher_is_better"
+    accepts_context: bool = field(init=False, repr=False, compare=False)
 
     def __post_init__(self) -> None:
         if not isinstance(self.name, str) or not self.name.strip():
             raise ValueError("scorer name must not be blank")
         if not callable(self.fn):
             raise ValueError("scorer fn must be callable")
+        object.__setattr__(self, "accepts_context", _accepts_context(self.fn))
         _validate_thresholds(
             threshold=self.threshold,
             pass_rate_threshold=self.pass_rate_threshold,
@@ -120,6 +149,33 @@ def to_criteria_wire(self) -> dict[str, Any]:
 type Criterion = Judge | Scorer
 
 
+def _accepts_context(fn: Callable[..., Any]) -> bool:
+    """Whether ``fn`` declares a third positional parameter for the context.
+
+    Raises ``ValueError`` when ``fn`` requires more than three arguments.
+    A function with no inspectable signature is called with two arguments.
+    """
+    try:
+        parameters = inspect.signature(fn).parameters.values()
+    except (TypeError, ValueError):
+        return False
+    positional = [
+        parameter
+        for parameter in parameters
+        if parameter.kind
+        in (
+            inspect.Parameter.POSITIONAL_ONLY,
+            inspect.Parameter.POSITIONAL_OR_KEYWORD,
+        )
+    ]
+    required = [
+        parameter for parameter in positional if parameter.default is parameter.empty
+    ]
+    if len(required) > 3:
+        raise ValueError("scorer fn must accept at most (row, output, context)")
+    return len(positional) >= 3
+
+
 def _validate_thresholds(
     *,
     threshold: float | None,

packages/client/src/launchdarkly_ai_server/evaluations/runner.py

@@ -33,7 +33,7 @@
     to_ld_context,
 )
 from .api import EvaluationsError, LDApiClient, LDApiError
-from .criteria import Criterion, Judge, Scorer
+from .criteria import Criterion, Judge, Scorer, ScorerContext
 from .events import (
     CriterionEventPayload,
     CriterionStatus,
@@ -859,7 +859,14 @@ async def _run_scorer_for_result(
             metadata=row.get("metadata"),
         )
         try:
-            score_value = scorer.fn(dataset_row, row.get("output"))
+            if scorer.accepts_context:
+                context = ScorerContext(
+                    tool_calls=tuple(row.get("tool_calls") or ()),
+                    tool_calls_omitted=int(row.get("tool_calls_omitted") or 0),
+                )
+                score_value = scorer.fn(dataset_row, row.get("output"), context)  # type: ignore[call-arg]
+            else:
+                score_value = scorer.fn(dataset_row, row.get("output"))  # type: ignore[call-arg]
             if inspect.isawaitable(score_value):
                 score_value = await score_value
         except Exception as error:

packages/client/tests/test_evaluations_run.py

@@ -16,8 +16,10 @@
     HttpResponse,
     Judge,
     Scorer,
+    ScorerContext,
     init_evaluations,
 )
+from launchdarkly_ai_server.trajectory import ToolInvocation
 
 
 @pytest.fixture(autouse=True)
@@ -2965,3 +2967,108 @@ def test_ai_config_variation_from_api_layers_the_model_config() -> None:
     unlinked = AIConfigVariation.from_api(latest)
     assert "provider" not in unlinked.generation
     assert unlinked.generation["parameters"] == {"temperature": 0.7}
+
+
+def _scorer_runner() -> Any:
+    from launchdarkly_ai_server.evaluations.api import LDApiClient
+    from launchdarkly_ai_server.evaluations.runner import EvaluationsRunner
+
+    return EvaluationsRunner(
+        LDApiClient(api_token="token", transport=failing_transport)
+    )
+
+
+def _generated_row(**overrides: Any) -> dict[str, Any]:
+    row: dict[str, Any] = {
+        "row_index": 0,
+        "input": "Question",
+        "expected_output": None,
+        "variables": {},
+        "metadata": None,
+        "output": "answer",
+        "status": "COMPLETE",
+        "tool_calls": [
+            ToolInvocation(name="lookup_order", arguments='{"id":"A1"}', result="ok"),
+            ToolInvocation(name="refund", arguments='{"id":"A1"}', result="done"),
+        ],
+        "tool_calls_omitted": 3,
+    }
+    row.update(overrides)
+    return row
+
+
+@pytest.mark.asyncio
+async def test_scorer_with_a_third_parameter_receives_the_tool_calls_in_order() -> None:
+    seen: list[ScorerContext] = []
+
+    def scorer_fn(row: DatasetRow, output: Any, context: ScorerContext) -> float:
+        seen.append(context)
+        names = [call.name for call in context.tool_calls]
+        return 1.0 if names == ["lookup_order", "refund"] else 0.0
+
+    result = await _scorer_runner()._run_scorer_for_result(
+        _generated_row(), Scorer(name="trajectory", fn=scorer_fn)
+    )
+
+    assert result["status"] == "COMPLETE"
+    assert result["score"] == 1.0
+    assert seen[0].tool_calls_omitted == 3
+
+
+@pytest.mark.asyncio
+async def test_async_scorer_with_a_context_is_awaited() -> None:
+    async def scorer_fn(row: DatasetRow, output: Any, context: ScorerContext) -> float:
+        return 1.0 if len(context.tool_calls) == 2 else 0.0
+
+    result = await _scorer_runner()._run_scorer_for_result(
+        _generated_row(), Scorer(name="trajectory", fn=scorer_fn)
+    )
+
+    assert result["score"] == 1.0
+
+
+@pytest.mark.asyncio
+async def test_scorer_context_is_empty_when_no_tool_was_called() -> None:
+    def scorer_fn(row: DatasetRow, output: Any, context: ScorerContext) -> float:
+        assert context.tool_calls == ()
+        assert context.tool_calls_omitted == 0
+        return 1.0
+
+    result = await _scorer_runner()._run_scorer_for_result(
+        _generated_row(tool_calls=[], tool_calls_omitted=0),
+        Scorer(name="trajectory", fn=scorer_fn),
+    )
+
+    assert result["status"] == "COMPLETE"
+
+
+@pytest.mark.asyncio
+async def test_two_parameter_scorer_is_called_without_a_context() -> None:
+    calls: list[tuple[Any, ...]] = []
+
+    def scorer_fn(*args: Any) -> float:
+        calls.append(args)
+        return 1.0
+
+    def two_parameter(row: DatasetRow, output: Any) -> float:
+        return scorer_fn(row, output)
+
+    await _scorer_runner()._run_scorer_for_result(
+        _generated_row(), Scorer(name="plain", fn=two_parameter)
+    )
+
+    assert len(calls[0]) == 2
+
+
+def test_scorer_detects_the_context_parameter_once_at_construction() -> None:
+    assert Scorer(name="a", fn=lambda row, output: 1.0).accepts_context is False
+    assert Scorer(name="b", fn=lambda row, output, ctx: 1.0).accepts_context is True
+    assert (
+        Scorer(name="c", fn=lambda row, output, ctx=None: 1.0).accepts_context is True
+    )
+    assert Scorer(name="d", fn=lambda *args: 1.0).accepts_context is False
+
+
+def test_scorer_rejects_a_function_that_needs_more_than_three_arguments() -> None:
+    with pytest.raises(ValueError, match="at most"):
+        Scorer(name="too-many", fn=lambda a, b, c, d: 1.0)