From 418a42b2566352f85fa6d06e097abd953dc92baf Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Tue, 16 Jun 2026 10:44:36 -0700 Subject: [PATCH] fix(eval): handle failed inference results without invocations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merge https://github.com/google/adk-python/pull/5878 ## What changed - Return a failed `EvalCaseResult` when inference failed before producing any invocations. - Preserve the existing session lookup path when a failed inference still has a session id. - Add a regression test for `InferenceResult(status=FAILURE, inferences=None)`. - Clean up two existing lint issues in the touched eval test file so the local changed-file ruff check passes. This prevents the eval runner from replacing the original inference error with `TypeError: object of type 'NoneType' has no len()`. Fixes #5876 ## To verify - `.\.venv\Scripts\python.exe -m py_compile src\google\adk\evaluation\local_eval_service.py tests\unittests\evaluation\test_local_eval_service.py` - `.\.venv\Scripts\python.exe -m pytest tests\unittests\evaluation\test_local_eval_service.py -k "failed_without_inferences or evaluate_single_inference_result" -q --basetemp .tmp\pytest` - `.\.venv\Scripts\python.exe -m ruff check src\google\adk\evaluation\local_eval_service.py tests\unittests\evaluation\test_local_eval_service.py` - `.\.venv\Scripts\python.exe -m pyink --check src\google\adk\evaluation\local_eval_service.py tests\unittests\evaluation\test_local_eval_service.py` - `git diff --check` Co-authored-by: George Weale COPYBARA_INTEGRATE_REVIEW=https://github.com/google/adk-python/pull/5878 from he-yufeng:fix/eval-failed-inference-none 5ed777d2939d03b891615da9e215b807536809ad PiperOrigin-RevId: 933176248 (cherry picked from commit 9a6cf60fa8d54523e95943ebdb49d4f35341aed0) --- .../adk/evaluation/local_eval_service.py | 23 ++++++++++++ .../evaluation/test_local_eval_service.py | 36 +++++++++++++++++-- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/google/adk/evaluation/local_eval_service.py b/src/google/adk/evaluation/local_eval_service.py index b749b7e8f19..5fba2c05e4f 100644 --- a/src/google/adk/evaluation/local_eval_service.py +++ b/src/google/adk/evaluation/local_eval_service.py @@ -270,6 +270,29 @@ async def _evaluate_single_inference_result( else 'test_user_id' ) + if inference_result.inferences is None: + session_details = None + if inference_result.session_id is not None: + session_details = await self._session_service.get_session( + app_name=inference_result.app_name, + user_id=user_id, + session_id=inference_result.session_id, + ) + return ( + inference_result, + EvalCaseResult( + eval_set_file=inference_result.eval_set_id, + eval_set_id=inference_result.eval_set_id, + eval_id=inference_result.eval_case_id, + final_eval_status=EvalStatus.FAILED, + overall_eval_metric_results=[], + eval_metric_result_per_invocation=[], + session_id=inference_result.session_id or '', + session_details=session_details, + user_id=user_id, + ), + ) + if eval_case.conversation_scenario is None and len( inference_result.inferences ) != len(eval_case.conversation): diff --git a/tests/unittests/evaluation/test_local_eval_service.py b/tests/unittests/evaluation/test_local_eval_service.py index 894fc27c077..adcf982507e 100644 --- a/tests/unittests/evaluation/test_local_eval_service.py +++ b/tests/unittests/evaluation/test_local_eval_service.py @@ -15,7 +15,6 @@ from __future__ import annotations import asyncio -import sys from typing import Optional from google.adk.agents.llm_agent import LlmAgent @@ -465,6 +464,39 @@ async def test_evaluate_single_inference_result( assert metric_result.eval_status == EvalStatus.PASSED +@pytest.mark.asyncio +async def test_evaluate_single_inference_result_failed_without_inferences( + eval_service, mock_eval_sets_manager, mocker +): + inference_result = InferenceResult( + app_name="test_app", + eval_set_id="test_eval_set", + eval_case_id="case1", + inferences=None, + session_id="session1", + status=InferenceStatus.FAILURE, + error_message="auth failed", + ) + eval_metric = EvalMetric(metric_name="fake_metric", threshold=0.5) + evaluate_config = EvaluateConfig(eval_metrics=[eval_metric], parallelism=1) + + mock_eval_case = mocker.MagicMock(spec=EvalCase) + mock_eval_case.conversation = [] + mock_eval_case.conversation_scenario = None + mock_eval_case.session_input = None + mock_eval_sets_manager.get_eval_case.return_value = mock_eval_case + + _, result = await eval_service._evaluate_single_inference_result( + inference_result=inference_result, evaluate_config=evaluate_config + ) + + assert result.eval_id == "case1" + assert result.session_id == "session1" + assert result.final_eval_status == EvalStatus.FAILED + assert result.overall_eval_metric_results == [] + assert result.eval_metric_result_per_invocation == [] + + @pytest.mark.asyncio async def test_evaluate_single_inference_result_for_conversation_scenario( eval_service, mock_eval_sets_manager, mocker @@ -520,7 +552,7 @@ async def test_evaluate_single_inference_result_for_conversation_scenario( for i in range(3): invocation_result = result.eval_metric_result_per_invocation[i] assert invocation_result.actual_invocation == inference_result.inferences[i] - assert invocation_result.expected_invocation == None + assert invocation_result.expected_invocation is None assert len(invocation_result.eval_metric_results) == 1 metric_result = invocation_result.eval_metric_results[0] assert metric_result.metric_name == "fake_single_sided_metric"