fix(batch-evaluation): separate stats for evaluators sharing a name

langfuse/batch_evaluation.py

@@ -571,6 +571,9 @@ class BatchEvaluationResult:
         total_composite_scores_created: Scores created by the composite evaluator.
         total_evaluations_failed: Number of individual evaluator failures across all items.
         evaluator_stats: List of per-evaluator statistics (success/failure rates, scores created).
+            One entry per evaluator, in the order the evaluators were passed. Names
+            can repeat when two evaluators share a name, so key on position rather
+            than `name` if you build a lookup from this list.
         resume_token: Token for continuing the run. Set when a batch fetch failed
             (`completed=False`) or when `max_items` was reached while more items
             may exist (`has_more_items=True`); None otherwise.
@@ -686,7 +689,8 @@ def __init__(
             total_scores_created: Scores from item-level evaluators.
             total_composite_scores_created: Scores from composite evaluator.
             total_evaluations_failed: Individual evaluator failures.
-            evaluator_stats: Per-evaluator statistics.
+            evaluator_stats: Per-evaluator statistics, one entry per evaluator in
+                order. Names can repeat.
             resume_token: Token for resuming (None if completed).
             completed: Whether all items were processed.
             duration_seconds: Total execution time.
@@ -894,11 +898,13 @@ async def run_async(
         item_evaluations: Dict[str, List[Evaluation]] = {}
         previously_processed = resume_from.items_processed if resume_from else 0
 
+        # Keyed by position: evaluators can share a __name__ (e.g. every
+        # create_evaluator_from_autoevals() result is "langfuse_evaluator").
         evaluator_stats_dict = {
-            getattr(evaluator, "__name__", "unknown_evaluator"): EvaluatorStats(
+            index: EvaluatorStats(
                 name=getattr(evaluator, "__name__", "unknown_evaluator")
             )
-            for evaluator in evaluators
+            for index, evaluator in enumerate(evaluators)
         }
 
         semaphore = asyncio.Semaphore(max_concurrency)
@@ -1138,7 +1144,7 @@ async def _process_batch_evaluation_item(
         composite_evaluator: Optional[CompositeEvaluatorFunction],
         metadata: Optional[Dict[str, Any]],
         _add_observation_scores_to_trace: bool,
-        evaluator_stats_dict: Dict[str, EvaluatorStats],
+        evaluator_stats_dict: Dict[int, EvaluatorStats],
     ) -> Tuple[int, int, int, List[Evaluation]]:
         """Process a single item: map, evaluate, create scores.
 
@@ -1150,7 +1156,7 @@ async def _process_batch_evaluation_item(
             metadata: Additional metadata to add to scores.
             _add_observation_scores_to_trace: Whether to duplicate
                 observation-level scores at trace level.
-            evaluator_stats_dict: Dictionary tracking evaluator statistics.
+            evaluator_stats_dict: Evaluator statistics keyed by evaluator position.
 
         Returns:
             Tuple of (scores_created, composite_scores_created, evaluations_failed, all_evaluations).
@@ -1169,9 +1175,9 @@ async def _process_batch_evaluation_item(
         evaluator_inputs = await self._run_mapper(mapper, item)
 
         evaluations: List[Evaluation] = []
-        for evaluator in evaluators:
+        for index, evaluator in enumerate(evaluators):
             evaluator_name = getattr(evaluator, "__name__", "unknown_evaluator")
-            stats = evaluator_stats_dict[evaluator_name]
+            stats = evaluator_stats_dict[index]
             stats.total_runs += 1
 
             try:
@@ -1459,7 +1465,7 @@ def _build_result(
         total_scores_created: int,
         total_composite_scores_created: int,
         total_evaluations_failed: int,
-        evaluator_stats_dict: Dict[str, EvaluatorStats],
+        evaluator_stats_dict: Dict[int, EvaluatorStats],
         resume_token: Optional[BatchEvaluationResumeToken],
         completed: bool,
         start_time: float,

tests/unit/test_batch_evaluation.py

@@ -17,7 +17,7 @@
     BatchEvaluationRunner,
     EvaluatorInputs,
 )
-from langfuse.experiment import Evaluation
+from langfuse.experiment import Evaluation, create_evaluator_from_autoevals
 
 BASE_TIME = datetime(2026, 1, 1, tzinfo=timezone.utc)
 
@@ -376,6 +376,60 @@ def composite(*, evaluations, **kwargs):
     assert client.create_score.call_count == 2
 
 
+@pytest.mark.asyncio
+async def test_evaluators_sharing_a_name_get_separate_stats():
+    runner, _, client = make_runner([make_observation(i) for i in range(3)])
+
+    def make_evaluator(*, fail: bool):
+        def evaluator(*, output, **kwargs):
+            if fail:
+                raise RuntimeError("boom")
+            return [Evaluation(name="a", value=1), Evaluation(name="b", value=2)]
+
+        return evaluator
+
+    result = await run(
+        runner, evaluators=[make_evaluator(fail=False), make_evaluator(fail=True)]
+    )
+
+    assert [
+        (s.name, s.total_runs, s.successful_runs, s.failed_runs, s.total_scores_created)
+        for s in result.evaluator_stats
+    ] == [("evaluator", 3, 3, 0, 6), ("evaluator", 3, 0, 3, 0)]
+    assert result.total_evaluations_failed == 3
+    assert client.create_score.call_count == 6
+
+
+@pytest.mark.asyncio
+async def test_autoevals_evaluators_get_separate_stats():
+    runner, _, _ = make_runner([make_observation(i) for i in range(2)])
+
+    class FakeScorer:
+        def __init__(self, name: str, score: float):
+            self.name = name
+            self.score = score
+
+        def __call__(self, **kwargs):
+            return SimpleNamespace(name=self.name, score=self.score, metadata=None)
+
+    evaluators = [
+        create_evaluator_from_autoevals(FakeScorer("Levenshtein", 0.5)),
+        create_evaluator_from_autoevals(FakeScorer("ExactMatch", 1.0)),
+    ]
+
+    result = await run(runner, evaluators=evaluators)
+
+    assert len(result.evaluator_stats) == 2
+    assert [(s.total_runs, s.total_scores_created) for s in result.evaluator_stats] == [
+        (2, 2),
+        (2, 2),
+    ]
+    assert [e.name for e in result.item_evaluations["obs-0"]] == [
+        "Levenshtein",
+        "ExactMatch",
+    ]
+
+
 @pytest.mark.asyncio
 async def test_async_mapper_and_evaluator_are_awaited():
     runner, _, _ = make_runner([make_observation(1)])