diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py index 9cd942d9..c970303b 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py @@ -9,7 +9,7 @@ Transport, urllib_transport, ) -from .criteria import Criterion, Judge, Scorer, SuccessDirection +from .criteria import Criterion, Judge, Scorer, ScorerContext, SuccessDirection from .module import EvaluationsModule, init_evaluations from .types import ( AIConfig, @@ -35,6 +35,7 @@ "LDApiError", "RunSummary", "Scorer", + "ScorerContext", "SuccessDirection", "Transport", "Usage", diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py index ee0962fd..565bcc91 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py @@ -1,13 +1,19 @@ from __future__ import annotations +import inspect from collections.abc import Awaitable, Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from math import isnan from typing import Any, Literal +from ..trajectory import ToolInvocation from .types import DatasetRow -type ScorerFn = Callable[[DatasetRow, Any], float | bool | Awaitable[float | bool]] +type ScorerResult = float | bool | Awaitable[float | bool] +type ScorerFn = ( + Callable[[DatasetRow, Any], ScorerResult] + | Callable[[DatasetRow, Any, ScorerContext], ScorerResult] +) type SuccessDirection = Literal["higher_is_better", "lower_is_better"] @@ -63,15 +69,36 @@ def to_criteria_wire(self) -> dict[str, Any]: } +@dataclass(frozen=True) +class ScorerContext: + """Extra inputs for a scorer, beyond the row and the generated output. + + A scorer receives this as an optional third argument. New inputs are added + as new fields, so the scorer signature does not change. + + ``tool_calls`` lists the tools the handler called while it produced the + output, in the order the calls started. It is empty when the handler + called no tool. At most 50 calls are recorded. ``tool_calls_omitted`` + counts the calls made past that limit. Each call records its arguments and + result as text, cut to 2000 characters. + """ + + tool_calls: tuple[ToolInvocation, ...] = () + tool_calls_omitted: int = 0 + + @dataclass(frozen=True) class Scorer: """Local deterministic scorer run for each generated evaluation row. ``fn`` may be sync or async and receives ``(row, output)``, where ``row`` is the :class:`~launchdarkly_ai_server.evaluations.types.DatasetRow` the - output was generated from and ``output`` is the generated output. It must - return a boolean or a numeric score from 0 to 1. Boolean results are - converted to 1.0 or 0.0 before being emitted as evaluation events. + output was generated from and ``output`` is the generated output. To + receive more inputs, declare a third positional parameter. It receives a + :class:`ScorerContext`. A function that declares only two parameters is + called with two arguments. ``fn`` must return a boolean or a numeric score + from 0 to 1. Boolean results are converted to 1.0 or 0.0 before being + emitted as evaluation events. ``threshold`` defaults to 1.0: a row passes only on a perfect score, which matches the common case of boolean scorers. Pass a lower threshold for @@ -90,12 +117,14 @@ class Scorer: threshold: float | None = 1.0 pass_rate_threshold: float | None = None success_direction: SuccessDirection = "higher_is_better" + accepts_context: bool = field(init=False, repr=False, compare=False) def __post_init__(self) -> None: if not isinstance(self.name, str) or not self.name.strip(): raise ValueError("scorer name must not be blank") if not callable(self.fn): raise ValueError("scorer fn must be callable") + object.__setattr__(self, "accepts_context", _accepts_context(self.fn)) _validate_thresholds( threshold=self.threshold, pass_rate_threshold=self.pass_rate_threshold, @@ -120,6 +149,33 @@ def to_criteria_wire(self) -> dict[str, Any]: type Criterion = Judge | Scorer +def _accepts_context(fn: Callable[..., Any]) -> bool: + """Whether ``fn`` declares a third positional parameter for the context. + + Raises ``ValueError`` when ``fn`` requires more than three arguments. + A function with no inspectable signature is called with two arguments. + """ + try: + parameters = inspect.signature(fn).parameters.values() + except (TypeError, ValueError): + return False + positional = [ + parameter + for parameter in parameters + if parameter.kind + in ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + ) + ] + required = [ + parameter for parameter in positional if parameter.default is parameter.empty + ] + if len(required) > 3: + raise ValueError("scorer fn must accept at most (row, output, context)") + return len(positional) >= 3 + + def _validate_thresholds( *, threshold: float | None, diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 6240e980..d9f9ce25 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -33,7 +33,7 @@ to_ld_context, ) from .api import EvaluationsError, LDApiClient, LDApiError -from .criteria import Criterion, Judge, Scorer +from .criteria import Criterion, Judge, Scorer, ScorerContext from .events import ( CriterionEventPayload, CriterionStatus, @@ -859,7 +859,14 @@ async def _run_scorer_for_result( metadata=row.get("metadata"), ) try: - score_value = scorer.fn(dataset_row, row.get("output")) + if scorer.accepts_context: + context = ScorerContext( + tool_calls=tuple(row.get("tool_calls") or ()), + tool_calls_omitted=int(row.get("tool_calls_omitted") or 0), + ) + score_value = scorer.fn(dataset_row, row.get("output"), context) # type: ignore[call-arg] + else: + score_value = scorer.fn(dataset_row, row.get("output")) # type: ignore[call-arg] if inspect.isawaitable(score_value): score_value = await score_value except Exception as error: diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 4a4b8571..9ce3d7b4 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -16,8 +16,10 @@ HttpResponse, Judge, Scorer, + ScorerContext, init_evaluations, ) +from launchdarkly_ai_server.trajectory import ToolInvocation @pytest.fixture(autouse=True) @@ -2965,3 +2967,108 @@ def test_ai_config_variation_from_api_layers_the_model_config() -> None: unlinked = AIConfigVariation.from_api(latest) assert "provider" not in unlinked.generation assert unlinked.generation["parameters"] == {"temperature": 0.7} + + +def _scorer_runner() -> Any: + from launchdarkly_ai_server.evaluations.api import LDApiClient + from launchdarkly_ai_server.evaluations.runner import EvaluationsRunner + + return EvaluationsRunner( + LDApiClient(api_token="token", transport=failing_transport) + ) + + +def _generated_row(**overrides: Any) -> dict[str, Any]: + row: dict[str, Any] = { + "row_index": 0, + "input": "Question", + "expected_output": None, + "variables": {}, + "metadata": None, + "output": "answer", + "status": "COMPLETE", + "tool_calls": [ + ToolInvocation(name="lookup_order", arguments='{"id":"A1"}', result="ok"), + ToolInvocation(name="refund", arguments='{"id":"A1"}', result="done"), + ], + "tool_calls_omitted": 3, + } + row.update(overrides) + return row + + +@pytest.mark.asyncio +async def test_scorer_with_a_third_parameter_receives_the_tool_calls_in_order() -> None: + seen: list[ScorerContext] = [] + + def scorer_fn(row: DatasetRow, output: Any, context: ScorerContext) -> float: + seen.append(context) + names = [call.name for call in context.tool_calls] + return 1.0 if names == ["lookup_order", "refund"] else 0.0 + + result = await _scorer_runner()._run_scorer_for_result( + _generated_row(), Scorer(name="trajectory", fn=scorer_fn) + ) + + assert result["status"] == "COMPLETE" + assert result["score"] == 1.0 + assert seen[0].tool_calls_omitted == 3 + + +@pytest.mark.asyncio +async def test_async_scorer_with_a_context_is_awaited() -> None: + async def scorer_fn(row: DatasetRow, output: Any, context: ScorerContext) -> float: + return 1.0 if len(context.tool_calls) == 2 else 0.0 + + result = await _scorer_runner()._run_scorer_for_result( + _generated_row(), Scorer(name="trajectory", fn=scorer_fn) + ) + + assert result["score"] == 1.0 + + +@pytest.mark.asyncio +async def test_scorer_context_is_empty_when_no_tool_was_called() -> None: + def scorer_fn(row: DatasetRow, output: Any, context: ScorerContext) -> float: + assert context.tool_calls == () + assert context.tool_calls_omitted == 0 + return 1.0 + + result = await _scorer_runner()._run_scorer_for_result( + _generated_row(tool_calls=[], tool_calls_omitted=0), + Scorer(name="trajectory", fn=scorer_fn), + ) + + assert result["status"] == "COMPLETE" + + +@pytest.mark.asyncio +async def test_two_parameter_scorer_is_called_without_a_context() -> None: + calls: list[tuple[Any, ...]] = [] + + def scorer_fn(*args: Any) -> float: + calls.append(args) + return 1.0 + + def two_parameter(row: DatasetRow, output: Any) -> float: + return scorer_fn(row, output) + + await _scorer_runner()._run_scorer_for_result( + _generated_row(), Scorer(name="plain", fn=two_parameter) + ) + + assert len(calls[0]) == 2 + + +def test_scorer_detects_the_context_parameter_once_at_construction() -> None: + assert Scorer(name="a", fn=lambda row, output: 1.0).accepts_context is False + assert Scorer(name="b", fn=lambda row, output, ctx: 1.0).accepts_context is True + assert ( + Scorer(name="c", fn=lambda row, output, ctx=None: 1.0).accepts_context is True + ) + assert Scorer(name="d", fn=lambda *args: 1.0).accepts_context is False + + +def test_scorer_rejects_a_function_that_needs_more_than_three_arguments() -> None: + with pytest.raises(ValueError, match="at most"): + Scorer(name="too-many", fn=lambda a, b, c, d: 1.0)