[Evaluation] Align shared evaluator utils/validators/base with azureml-assets builtin

sdk/evaluation/azure-ai-evaluation/CHANGELOG.md

@@ -10,6 +10,11 @@
   `QAEvaluator`, `RougeScoreEvaluator`, `DocumentRetrievalEvaluator`, the task navigation efficiency
   evaluator, and the shared evaluator base (conversation/tool-call input validation).
 
+- Fixed OpenAPI tool-call validation in tool evaluators (e.g. `ToolCallAccuracyEvaluator`). OpenAPI
+  tool definitions are expanded into their nested functions when present, so tool calls referencing a
+  nested function name validate correctly, while OpenAPI tool definitions without nested functions are
+  kept as is so tool calls referencing the top-level tool name continue to validate.
+
 - Fixed `RedTeam.scan()` storing decoded plaintext instead of the actual
   encoded payload for converter-based attack strategies (Base64, Flip,
   Morse, ROT13, etc.) in `evaluation_results.json` / `results.json`. The
@@ -22,6 +27,19 @@
   (non-encoded) strategies are unaffected.
   Resolves [#47228](https://github.com/Azure/azure-sdk-for-python/issues/47228).
 
+### Other Changes
+
+- Conversation/tool message preprocessing now normalizes `openapi_call` / `openapi_call_output` content
+  items to `tool_call` / `tool_result` (previously only `function_call` / `function_call_output` were
+  normalized), so evaluators correctly handle OpenAPI-tool agent transcripts.
+- Tool results are now serialized as JSON when rendering agent responses for LLM-judge evaluators, so
+  list/dict outputs from grounding tools (Azure AI Search, SharePoint, Fabric) are readable instead of a
+  Python `repr`. Plain string tool results are unchanged.
+- Evaluators no longer log raw customer payloads in fallback/debug paths. `reformat_agent_response`,
+  `reformat_conversation_history`, `reformat_tool_definitions`, the tool-call-success reformat helpers,
+  and Groundedness context extraction now emit structural summaries only (via `_log_safe_summary`),
+  never raw query/response/tool payloads.
+
 ## 1.17.0 (2026-06-03)
 
 ### Breaking Changes

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py

@@ -6,7 +6,6 @@
 import math
 import re
 import os
-from itertools import chain
 from typing import Dict, Optional, TypeVar, Union, List
 
 if os.getenv("AI_EVALS_USE_PF_PROMPTY", "false").lower() == "true":
@@ -76,20 +75,29 @@ def _drop_mcp_approval_messages(messages):
 
 
 def _normalize_function_call_types(messages):
-    """Normalize function_call/function_call_output types to tool_call/tool_result."""
+    """Normalize function_call/function_call_output/openapi_call/openapi_call_output types to tool_call/tool_result."""
     if not isinstance(messages, list):
         return messages
     for msg in messages:
         if isinstance(msg, dict) and isinstance(msg.get("content"), list):
             for item in msg["content"]:
-                if isinstance(item, dict) and item.get("type") == "function_call":
+                if not isinstance(item, dict):
+                    continue
+                item_type = item.get("type")
+                if item_type == "function_call":
                     item["type"] = "tool_call"
                     if "function_call" in item:
                         item["tool_call"] = item.pop("function_call")
-                elif isinstance(item, dict) and item.get("type") == "function_call_output":
+                elif item_type == "function_call_output":
                     item["type"] = "tool_result"
                     if "function_call_output" in item:
                         item["tool_result"] = item.pop("function_call_output")
+                elif item_type == "openapi_call":
+                    item["type"] = "tool_call"
+                elif item_type == "openapi_call_output":
+                    item["type"] = "tool_result"
+                    if "openapi_call_output" in item:
+                        item["tool_result"] = item.pop("openapi_call_output")
     return messages
 
 
@@ -370,13 +378,18 @@ def _extract_needed_tool_definitions(
         built_in_definitions = self._get_needed_built_in_tool_definitions(tool_calls)
         needed_tool_definitions.extend(built_in_definitions)
 
-        # OpenAPI tool is a collection of functions, so we need to expand it
-        tool_definitions_expanded = list(
-            chain.from_iterable(
-                tool.get("functions", []) if tool.get("type") == "openapi" else [tool]
-                for tool in needed_tool_definitions
-            )
-        )
+        # An OpenAPI tool is a collection of functions. When an OpenAPI tool
+        # definition exposes its nested functions, expand it into those functions
+        # so tool calls referencing a nested function name can be matched. If an
+        # OpenAPI tool definition has no nested functions, keep the OpenAPI tool
+        # definition as is for backward compatibility.
+        tool_definitions_expanded = []
+        for tool in needed_tool_definitions:
+            openapi_functions = tool.get("functions") if tool.get("type") == "openapi" else None
+            if openapi_functions:
+                tool_definitions_expanded.extend(openapi_functions)
+            else:
+                tool_definitions_expanded.append(tool)
 
         # Validate that all tool calls have corresponding definitions
         for tool_call in tool_calls:

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py

@@ -399,7 +399,7 @@ def _get_context_from_agent_response(self, response, tool_definitions):
         try:
             logger.debug("Extracting context from response")
             tool_calls = self._parse_tools_from_response(response=response)
-            logger.debug(f"Tool Calls parsed successfully: {tool_calls}")
+            logger.debug("Tool calls parsed successfully: count=%d", len(tool_calls) if tool_calls else 0)
 
             if not tool_calls:
                 return NO_CONTEXT

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py

@@ -14,6 +14,7 @@
 )
 from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
 from azure.ai.evaluation._evaluators._common._validators import ToolDefinitionsValidator, ValidatorInterface
+from azure.ai.evaluation._common.utils import _log_safe_summary, _stringify_tool_result
 from azure.ai.evaluation._common._experimental import experimental
 
 logger = logging.getLogger(__name__)
@@ -283,7 +284,7 @@ def _get_tool_calls_results(agent_response_msgs):
             for content in msg.get("content", []):
                 if content.get("type") == "tool_result":
                     result = content.get("tool_result")
-                    tool_results[msg["tool_call_id"]] = f"[TOOL_RESULT] {result}"
+                    tool_results[msg["tool_call_id"]] = f"[TOOL_RESULT] {_stringify_tool_result(result)}"
 
     # Second pass: parse assistant messages and tool calls
     for msg in agent_response_msgs:
@@ -320,18 +321,23 @@ def _reformat_tool_calls_results(response, logger=None):
             # fallback to the original response in that case
             if logger:
                 logger.warning(
-                    f"Empty agent response extracted, likely due to input schema change. "
-                    f"Falling back to using the original response"
+                    "Empty agent response extracted, likely due to input schema change. "
+                    "Falling back to using the original response. %s",
+                    _log_safe_summary(response),
                 )
             return response
         return "\n".join(agent_response)
-    except Exception:
+    except Exception as e:  # pylint: disable=broad-except
         # If the agent response cannot be parsed for whatever
         # reason (e.g. the converter format changed), the original response is returned
         # This is a fallback to ensure that the evaluation can still proceed.
         # See comments on reformat_conversation_history for more details.
         if logger:
-            logger.warning(f"Agent response could not be parsed, falling back to original response")
+            logger.warning(
+                "Agent response could not be parsed, falling back to original response. Error: %s. %s",
+                e,
+                _log_safe_summary(response),
+            )
         return response
 
 
@@ -345,12 +351,14 @@ def _reformat_tool_definitions(tool_definitions, logger=None):
             param_names = ", ".join(params.keys()) if params else "no parameters"
             output_lines.append(f"- {name}: {desc} (inputs: {param_names})")
         return "\n".join(output_lines)
-    except Exception:
+    except Exception as e:  # pylint: disable=broad-except
         # If the tool definitions cannot be parsed for whatever reason, the original tool definitions are returned
         # This is a fallback to ensure that the evaluation can still proceed.
         # See comments on reformat_conversation_history for more details.
         if logger:
             logger.warning(
-                f"Tool definitions could not be parsed, falling back to original definitions: {tool_definitions}"
+                "Tool definitions could not be parsed; falling back to raw definitions. Input shape: %s. Error: %s",
+                _log_safe_summary(tool_definitions),
+                e,
             )
         return tool_definitions

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_exceptions.py

@@ -107,6 +107,10 @@ class ErrorTarget(Enum):
     QA_EVALUATOR = "QAEvaluator"
     ROUGE_EVALUATOR = "RougeScoreEvaluator"
     DOCUMENT_RETRIEVAL_EVALUATOR = "DocumentRetrievalEvaluator"
+    QUALITY_GRADER_EVALUATOR = "QualityGraderEvaluator"
+    CUSTOMER_SATISFACTION_EVALUATOR = "CustomerSatisfactionEvaluator"
+    DEFLECTION_RATE_EVALUATOR = "DeflectionRateEvaluator"
+    REGEX_MATCH_EVALUATOR = "RegexMatchEvaluator"
 
 
 class EvaluationException(AzureError):

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_multi_turn_utilities.py

@@ -804,5 +804,22 @@ def test_drops_mcp_and_normalizes(self):
         # function_call_output should be normalized to tool_result
         assert result[2]["content"][0]["type"] == "tool_result"
 
+    def test_normalizes_openapi_types(self):
+        messages = [
+            {"role": "user", "content": [{"type": "text", "text": "Call the API"}]},
+            {
+                "role": "assistant",
+                "content": [{"type": "openapi_call", "name": "api", "arguments": {}, "tool_call_id": "c1"}],
+            },
+            {
+                "role": "tool",
+                "content": [{"type": "openapi_call_output", "openapi_call_output": "api result"}],
+            },
+        ]
+        result = _preprocess_messages(messages)
+        assert result[1]["content"][0]["type"] == "tool_call"
+        assert result[2]["content"][0]["type"] == "tool_result"
+        assert result[2]["content"][0]["tool_result"] == "api result"
+
 
 # endregion

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_accuracy_evaluator.py

@@ -2,7 +2,7 @@
 
 import pytest
 from azure.ai.evaluation import ToolCallAccuracyEvaluator
-from azure.ai.evaluation._exceptions import EvaluationException
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorTarget
 
 
 # This mock should return a dictionary that mimics the output of the prompty (the _flow call),
@@ -733,6 +733,139 @@ def test_evaluate_open_api_with_tool_definition(self, mock_model_config):
         assert result[f"{key}_score"] == 5.0
         assert result[f"{key}_passed"] is True
 
+    def test_extract_needed_tool_definitions_openapi_with_functions(self, mock_model_config):
+        """OpenAPI tool definitions that expose nested functions should be expanded so a
+        tool call referencing a nested function name is validated successfully."""
+        evaluator = ToolCallAccuracyEvaluator(model_config=mock_model_config)
+
+        tool_calls = [
+            {
+                "type": "tool_call",
+                "tool_call_id": "call_1",
+                "name": "get_countries_LookupCountryByCurrency",
+                "arguments": {"currency": "GBP"},
+            }
+        ]
+        tool_definitions = [
+            {
+                "name": "get_countries",
+                "type": "openapi",
+                "description": "Retrieve a list of countries",
+                "functions": [
+                    {
+                        "name": "get_countries_LookupCountryByCurrency",
+                        "type": "function",
+                        "description": "Search by currency.",
+                        "parameters": {
+                            "type": "object",
+                            "properties": {
+                                "currency": {"type": "string", "description": "The currency to search for."}
+                            },
+                            "required": ["currency"],
+                        },
+                    }
+                ],
+            }
+        ]
+
+        needed_tool_definitions = evaluator._extract_needed_tool_definitions(
+            tool_calls, tool_definitions, ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR
+        )
+
+        # The nested function name resolved during validation (no exception was raised),
+        # and the original OpenAPI tool definition is returned unchanged.
+        assert needed_tool_definitions == tool_definitions
+
+    def test_extract_needed_tool_definitions_openapi_without_functions(self, mock_model_config):
+        """OpenAPI tool definitions without nested functions should be kept as is so a
+        tool call referencing the top-level OpenAPI tool name is validated successfully."""
+        evaluator = ToolCallAccuracyEvaluator(model_config=mock_model_config)
+
+        tool_calls = [
+            {
+                "type": "tool_call",
+                "tool_call_id": "call_1",
+                "name": "get_countries",
+                "arguments": {"currency": "GBP"},
+            }
+        ]
+        tool_definitions = [
+            {
+                "name": "get_countries",
+                "type": "openapi",
+                "description": "Retrieve a list of countries",
+            }
+        ]
+
+        needed_tool_definitions = evaluator._extract_needed_tool_definitions(
+            tool_calls, tool_definitions, ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR
+        )
+
+        # The top-level OpenAPI tool name resolved during validation (no exception was raised).
+        assert needed_tool_definitions == tool_definitions
+
+    def test_extract_needed_tool_definitions_openapi_empty_functions(self, mock_model_config):
+        """OpenAPI tool definitions with an empty functions list should fall back to the
+        top-level OpenAPI tool definition for validation."""
+        evaluator = ToolCallAccuracyEvaluator(model_config=mock_model_config)
+
+        tool_calls = [
+            {
+                "type": "tool_call",
+                "tool_call_id": "call_1",
+                "name": "get_countries",
+                "arguments": {"currency": "GBP"},
+            }
+        ]
+        tool_definitions = [
+            {
+                "name": "get_countries",
+                "type": "openapi",
+                "description": "Retrieve a list of countries",
+                "functions": [],
+            }
+        ]
+
+        needed_tool_definitions = evaluator._extract_needed_tool_definitions(
+            tool_calls, tool_definitions, ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR
+        )
+
+        assert needed_tool_definitions == tool_definitions
+
+    def test_extract_needed_tool_definitions_openapi_missing_nested_function_raises(self, mock_model_config):
+        """A tool call referencing a nested function not present in an expanded OpenAPI tool
+        definition should still raise an EvaluationException."""
+        evaluator = ToolCallAccuracyEvaluator(model_config=mock_model_config)
+
+        tool_calls = [
+            {
+                "type": "tool_call",
+                "tool_call_id": "call_1",
+                "name": "get_countries_UnknownFunction",
+                "arguments": {"currency": "GBP"},
+            }
+        ]
+        tool_definitions = [
+            {
+                "name": "get_countries",
+                "type": "openapi",
+                "functions": [
+                    {
+                        "name": "get_countries_LookupCountryByCurrency",
+                        "type": "function",
+                        "parameters": {"type": "object", "properties": {}},
+                    }
+                ],
+            }
+        ]
+
+        with pytest.raises(EvaluationException) as exc_info:
+            evaluator._extract_needed_tool_definitions(
+                tool_calls, tool_definitions, ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR
+            )
+
+        assert "Tool definition for get_countries_UnknownFunction not found" in str(exc_info.value)
+
     def test_evaluate_missing_query(self, mock_model_config):
         """Test that evaluator raises exception when query is None or missing."""
         evaluator = ToolCallAccuracyEvaluator(model_config=mock_model_config)