From 4bbe63b2ef8e518390528d63d7033c7ef509ad6e Mon Sep 17 00:00:00 2001 From: Nafay Rizwani <80161736+Nafay-0@users.noreply.github.com> Date: Sat, 7 Nov 2026 22:34:44 +0200 Subject: [PATCH 1/2] fix(batch-evaluation): separate stats for same-named evaluators --- langfuse/batch_evaluation.py | 16 +++++---- tests/unit/test_batch_evaluation.py | 56 ++++++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/langfuse/batch_evaluation.py b/langfuse/batch_evaluation.py index 8ef89c79b..eb5921354 100644 --- a/langfuse/batch_evaluation.py +++ b/langfuse/batch_evaluation.py @@ -894,11 +894,13 @@ async def run_async( item_evaluations: Dict[str, List[Evaluation]] = {} previously_processed = resume_from.items_processed if resume_from else 0 + # Keyed by position: evaluators can share a __name__ (e.g. every + # create_evaluator_from_autoevals() result is "langfuse_evaluator"). evaluator_stats_dict = { - getattr(evaluator, "__name__", "unknown_evaluator"): EvaluatorStats( + index: EvaluatorStats( name=getattr(evaluator, "__name__", "unknown_evaluator") ) - for evaluator in evaluators + for index, evaluator in enumerate(evaluators) } semaphore = asyncio.Semaphore(max_concurrency) @@ -1138,7 +1140,7 @@ async def _process_batch_evaluation_item( composite_evaluator: Optional[CompositeEvaluatorFunction], metadata: Optional[Dict[str, Any]], _add_observation_scores_to_trace: bool, - evaluator_stats_dict: Dict[str, EvaluatorStats], + evaluator_stats_dict: Dict[int, EvaluatorStats], ) -> Tuple[int, int, int, List[Evaluation]]: """Process a single item: map, evaluate, create scores. @@ -1150,7 +1152,7 @@ async def _process_batch_evaluation_item( metadata: Additional metadata to add to scores. _add_observation_scores_to_trace: Whether to duplicate observation-level scores at trace level. - evaluator_stats_dict: Dictionary tracking evaluator statistics. + evaluator_stats_dict: Evaluator statistics keyed by evaluator position. Returns: Tuple of (scores_created, composite_scores_created, evaluations_failed, all_evaluations). @@ -1169,9 +1171,9 @@ async def _process_batch_evaluation_item( evaluator_inputs = await self._run_mapper(mapper, item) evaluations: List[Evaluation] = [] - for evaluator in evaluators: + for index, evaluator in enumerate(evaluators): evaluator_name = getattr(evaluator, "__name__", "unknown_evaluator") - stats = evaluator_stats_dict[evaluator_name] + stats = evaluator_stats_dict[index] stats.total_runs += 1 try: @@ -1459,7 +1461,7 @@ def _build_result( total_scores_created: int, total_composite_scores_created: int, total_evaluations_failed: int, - evaluator_stats_dict: Dict[str, EvaluatorStats], + evaluator_stats_dict: Dict[int, EvaluatorStats], resume_token: Optional[BatchEvaluationResumeToken], completed: bool, start_time: float, diff --git a/tests/unit/test_batch_evaluation.py b/tests/unit/test_batch_evaluation.py index 5e2d7c7c1..3356d4622 100644 --- a/tests/unit/test_batch_evaluation.py +++ b/tests/unit/test_batch_evaluation.py @@ -17,7 +17,7 @@ BatchEvaluationRunner, EvaluatorInputs, ) -from langfuse.experiment import Evaluation +from langfuse.experiment import Evaluation, create_evaluator_from_autoevals BASE_TIME = datetime(2026, 1, 1, tzinfo=timezone.utc) @@ -376,6 +376,60 @@ def composite(*, evaluations, **kwargs): assert client.create_score.call_count == 2 +@pytest.mark.asyncio +async def test_evaluators_sharing_a_name_get_separate_stats(): + runner, _, client = make_runner([make_observation(i) for i in range(3)]) + + def make_evaluator(*, fail: bool): + def evaluator(*, output, **kwargs): + if fail: + raise RuntimeError("boom") + return [Evaluation(name="a", value=1), Evaluation(name="b", value=2)] + + return evaluator + + result = await run( + runner, evaluators=[make_evaluator(fail=False), make_evaluator(fail=True)] + ) + + assert [ + (s.name, s.total_runs, s.successful_runs, s.failed_runs, s.total_scores_created) + for s in result.evaluator_stats + ] == [("evaluator", 3, 3, 0, 6), ("evaluator", 3, 0, 3, 0)] + assert result.total_evaluations_failed == 3 + assert client.create_score.call_count == 6 + + +@pytest.mark.asyncio +async def test_autoevals_evaluators_get_separate_stats(): + runner, _, _ = make_runner([make_observation(i) for i in range(2)]) + + class FakeScorer: + def __init__(self, name: str, score: float): + self.name = name + self.score = score + + def __call__(self, **kwargs): + return SimpleNamespace(name=self.name, score=self.score, metadata=None) + + evaluators = [ + create_evaluator_from_autoevals(FakeScorer("Levenshtein", 0.5)), + create_evaluator_from_autoevals(FakeScorer("ExactMatch", 1.0)), + ] + + result = await run(runner, evaluators=evaluators) + + assert len(result.evaluator_stats) == 2 + assert [(s.total_runs, s.total_scores_created) for s in result.evaluator_stats] == [ + (2, 2), + (2, 2), + ] + assert [e.name for e in result.item_evaluations["obs-0"]] == [ + "Levenshtein", + "ExactMatch", + ] + + @pytest.mark.asyncio async def test_async_mapper_and_evaluator_are_awaited(): runner, _, _ = make_runner([make_observation(1)]) From 9a40b27c9a4bd4faad2893f6288034e1f8f9035f Mon Sep 17 00:00:00 2001 From: Nafay Rizwani <80161736+Nafay-0@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:16:48 +0200 Subject: [PATCH 2/2] docs: note that evaluator_stats names can repeat --- langfuse/batch_evaluation.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/langfuse/batch_evaluation.py b/langfuse/batch_evaluation.py index eb5921354..249af320a 100644 --- a/langfuse/batch_evaluation.py +++ b/langfuse/batch_evaluation.py @@ -571,6 +571,9 @@ class BatchEvaluationResult: total_composite_scores_created: Scores created by the composite evaluator. total_evaluations_failed: Number of individual evaluator failures across all items. evaluator_stats: List of per-evaluator statistics (success/failure rates, scores created). + One entry per evaluator, in the order the evaluators were passed. Names + can repeat when two evaluators share a name, so key on position rather + than `name` if you build a lookup from this list. resume_token: Token for continuing the run. Set when a batch fetch failed (`completed=False`) or when `max_items` was reached while more items may exist (`has_more_items=True`); None otherwise. @@ -686,7 +689,8 @@ def __init__( total_scores_created: Scores from item-level evaluators. total_composite_scores_created: Scores from composite evaluator. total_evaluations_failed: Individual evaluator failures. - evaluator_stats: Per-evaluator statistics. + evaluator_stats: Per-evaluator statistics, one entry per evaluator in + order. Names can repeat. resume_token: Token for resuming (None if completed). completed: Whether all items were processed. duration_seconds: Total execution time.