diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 6843e2b2345c..7452fc7413be 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -10,6 +10,11 @@ `QAEvaluator`, `RougeScoreEvaluator`, `DocumentRetrievalEvaluator`, the task navigation efficiency evaluator, and the shared evaluator base (conversation/tool-call input validation). +- Fixed OpenAPI tool-call validation in tool evaluators (e.g. `ToolCallAccuracyEvaluator`). OpenAPI + tool definitions are expanded into their nested functions when present, so tool calls referencing a + nested function name validate correctly, while OpenAPI tool definitions without nested functions are + kept as is so tool calls referencing the top-level tool name continue to validate. + - Fixed `RedTeam.scan()` storing decoded plaintext instead of the actual encoded payload for converter-based attack strategies (Base64, Flip, Morse, ROT13, etc.) in `evaluation_results.json` / `results.json`. The @@ -22,6 +27,19 @@ (non-encoded) strategies are unaffected. Resolves [#47228](https://github.com/Azure/azure-sdk-for-python/issues/47228). +### Other Changes + +- Conversation/tool message preprocessing now normalizes `openapi_call` / `openapi_call_output` content + items to `tool_call` / `tool_result` (previously only `function_call` / `function_call_output` were + normalized), so evaluators correctly handle OpenAPI-tool agent transcripts. +- Tool results are now serialized as JSON when rendering agent responses for LLM-judge evaluators, so + list/dict outputs from grounding tools (Azure AI Search, SharePoint, Fabric) are readable instead of a + Python `repr`. Plain string tool results are unchanged. +- Evaluators no longer log raw customer payloads in fallback/debug paths. `reformat_agent_response`, + `reformat_conversation_history`, `reformat_tool_definitions`, the tool-call-success reformat helpers, + and Groundedness context extraction now emit structural summaries only (via `_log_safe_summary`), + never raw query/response/tool payloads. + ## 1.17.0 (2026-06-03) ### Breaking Changes diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py index d0e7f8e0f77e..b54677cf1b13 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py @@ -1,6 +1,7 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- +import json import os import posixpath import re @@ -649,7 +650,11 @@ def filter_to_used_tools(tool_definitions, msgs_lists, logger=None): return tool_definitions -def _get_conversation_history(query, include_system_messages=False, include_tool_messages=False): +def _get_conversation_history( + query, include_system_messages=False, include_tool_messages=False, include_tool_calls=False +): + if include_tool_calls: + return _get_conversation_history_with_tool_calls(query, include_system_messages=include_system_messages) all_user_queries, all_agent_responses = [], [] cur_user_query, cur_agent_response = [], [] system_message = None @@ -728,16 +733,19 @@ def _pretty_format_conversation_history(conversation_history): return formatted_history -def reformat_conversation_history(query, logger=None, include_system_messages=False, include_tool_messages=False): +def reformat_conversation_history( + query, logger=None, include_system_messages=False, include_tool_messages=False, include_tool_calls=False +): """Reformats the conversation history to a more compact representation.""" try: conversation_history = _get_conversation_history( query, include_system_messages=include_system_messages, include_tool_messages=include_tool_messages, + include_tool_calls=include_tool_calls, ) return _pretty_format_conversation_history(conversation_history) - except Exception as e: + except Exception as e: # pylint: disable=broad-except # If the conversation history cannot be parsed for whatever reason (e.g. the converter format changed), the original query is returned # This is a fallback to ensure that the evaluation can still proceed. However the accuracy of the evaluation will be affected. # From our tests the negative impact on IntentResolution is: @@ -746,7 +754,12 @@ def reformat_conversation_history(query, logger=None, include_system_messages=Fa # Lower percentage of mode in Likert scale (73.4% vs 75.4%) # Lower pairwise agreement between LLMs (85% vs 90% at the pass/fail level with threshold of 3) if logger: - logger.warning("Conversation history could not be parsed, falling back to original query") + logger.warning( + "Conversation history could not be parsed; falling back to raw input. " + "Evaluator accuracy will degrade. Input shape: %s. Error: %s", + _log_safe_summary(query), + e, + ) return query @@ -770,7 +783,7 @@ def _get_agent_response(agent_response_msgs, include_tool_messages=False): for content in msg.get("content", []): if content.get("type") == "tool_result": result = content.get("tool_result") - tool_results[msg["tool_call_id"]] = f"[TOOL_RESULT] {result}" + tool_results[msg["tool_call_id"]] = f"[TOOL_RESULT] {_stringify_tool_result(result)}" # Second pass: parse assistant messages and tool calls for msg in agent_response_msgs: @@ -808,16 +821,22 @@ def reformat_agent_response(response, logger=None, include_tool_messages=False): if agent_response == []: # If no message could be extracted, likely the format changed, fallback to the original response in that case if logger: - logger.debug( - "Empty agent response extracted, likely due to input schema change. Falling back to original response" + logger.warning( + "Empty agent response extracted, likely due to input schema change. " + "Falling back to original response. %s", + _log_safe_summary(response), ) return response return "\n".join(agent_response) - except Exception: + except Exception as e: # pylint: disable=broad-except # If the agent response cannot be parsed for whatever reason (e.g. the converter format changed), the original response is returned # This is a fallback to ensure that the evaluation can still proceed. See comments on reformat_conversation_history for more details. if logger: - logger.debug("Agent response could not be parsed, falling back to original response") + logger.warning( + "Agent response could not be parsed, falling back to original response. Error: %s. %s", + e, + _log_safe_summary(response), + ) return response @@ -831,11 +850,15 @@ def reformat_tool_definitions(tool_definitions, logger=None): param_names = ", ".join(params.keys()) if params else "no parameters" output_lines.append(f"- {name}: {desc} (inputs: {param_names})") return "\n".join(output_lines) - except Exception as e: + except Exception as e: # pylint: disable=broad-except # If the tool definitions cannot be parsed for whatever reason, the original tool definitions are returned # This is a fallback to ensure that the evaluation can still proceed. See comments on reformat_conversation_history for more details. if logger: - logger.debug("Tool definitions could not be parsed, falling back to original definitions") + logger.warning( + "Tool definitions could not be parsed; falling back to raw definitions. " "Input shape: %s. Error: %s", + _log_safe_summary(tool_definitions), + e, + ) return tool_definitions @@ -907,6 +930,354 @@ def simplify_messages(messages, drop_system=True, drop_tool_calls=False, logger= return messages +# Runtime tool-call statuses that indicate a failed or incomplete execution. +_FAILED_RUNTIME_STATUSES = frozenset({"failed", "incomplete"}) + + +def _stringify_tool_result(result): + """Render a tool_result value as a string the LLM judge can read. + + Tool outputs arrive in mixed shapes depending on the producer: function/MCP tools usually + emit a plain ``str``, while built-in grounding tools (``azure_ai_search``, ``azure_fabric``, + ``sharepoint_grounding``) emit a list/dict. Falling back to ``f"{result}"`` for the latter + produced a Python ``repr`` (single quotes, trailing commas) that the LLM had to + reverse-engineer. Strings are passed through unchanged (zero behavior change for function/MCP + tools), anything else is serialized as JSON with ``default=str`` so non-JSON-native values do + not raise, and ``None`` renders as the empty string. + + :param result: The raw tool_result value. + :type result: Any + :return: A string representation suitable for an LLM prompt. + :rtype: str + """ + if result is None: + return "" + if isinstance(result, str): + return result + try: + return json.dumps(result, default=str, ensure_ascii=False) + except (TypeError, ValueError): + return str(result) + + +def _log_safe_summary(obj): + """Return a non-sensitive structural summary of a payload for safe logging. + + The raw payload may contain customer-controlled data (tool arguments, tool results, assistant + text, database rows, file content, etc.) which can include credentials or PII. Logging the + payload itself risks leaking that data into telemetry sinks at any log level. This helper + returns shape-only metadata - type, length, top-level keys/roles - which is sufficient to + diagnose schema drift without exposing values. + + :param obj: The payload to summarize. + :type obj: Any + :return: A shape-only, non-sensitive summary string. + :rtype: str + """ + try: + type_name = type(obj).__name__ + if isinstance(obj, list): + roles = [] + for item in obj[:10]: + if isinstance(item, dict): + role = item.get("role") + if isinstance(role, str): + roles.append(role) + roles_summary = roles if roles else "n/a" + return f"type={type_name} len={len(obj)} roles={roles_summary}" + if isinstance(obj, dict): + keys = sorted(k for k in obj.keys() if isinstance(k, str))[:10] + return f"type={type_name} top_keys={keys}" + length = len(obj) if hasattr(obj, "__len__") else "n/a" + return f"type={type_name} len={length}" + except Exception: # pylint: disable=broad-except + return f"type={type(obj).__name__} (summary unavailable)" + + +def _coerce_bool(value) -> Optional[bool]: + """Coerce an LLM output value to bool or None. + + Handles Python booleans and string variants like 'true', 'false'. + + :param value: The value to coerce. + :type value: Any + :return: The coerced boolean, or None if it cannot be interpreted. + :rtype: Optional[bool] + """ + if isinstance(value, bool): + return value + if value is None: + return None + if isinstance(value, str): + lower = value.strip().lower() + if lower == "true": + return True + if lower == "false": + return False + return None + + +def _coerce_number(value) -> Optional[float]: + """Coerce an LLM output value to a number or None. + + Handles Python ints/floats and string variants like '3', '2.5', 'null'. + + :param value: The value to coerce. + :type value: Any + :return: The coerced number, or None if it cannot be interpreted. + :rtype: Optional[float] + """ + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return value + if value is None: + return None + if isinstance(value, str): + stripped = value.strip() + if stripped.lower() in ("null", "none", ""): + return None + try: + return float(stripped) + except ValueError: + return None + return None + + +def _collect_failed_tool_calls(messages): + """Return ordered, unique tool names whose runtime status indicates failure. + + A tool call is treated as a runtime failure when either its assistant ``tool_call`` content + block or its matched tool ``tool_result`` content block carries a ``status`` field in + ``{failed, incomplete}``. This lets callers short-circuit deterministically and skip the LLM + judge on the failure path. When the failing block carries no resolvable function name, the + tool ``tool_call_id`` is used as a stable identifier instead. + + :param messages: The list of conversation messages to scan. + :type messages: Any + :return: Ordered, de-duplicated list of failed tool names (or ids). + :rtype: List + """ + if not isinstance(messages, list): + return [] + + id_to_name = {} + failed_ids = [] + failed_names_without_id = [] + + for msg in messages: + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + for content in msg.get("content", []) or []: + if not isinstance(content, dict) or content.get("type") != "tool_call": + continue + if "tool_call" in content and "function" in content.get("tool_call", {}): + tc = content["tool_call"] + name = tc.get("function", {}).get("name", "") or "" + call_id = tc.get("id") + else: + name = content.get("name", "") or "" + call_id = content.get("tool_call_id") + if call_id is not None: + id_to_name[call_id] = name + status = content.get("status") + if isinstance(status, str) and status in _FAILED_RUNTIME_STATUSES: + if call_id is not None: + failed_ids.append(call_id) + elif name: + failed_names_without_id.append(name) + + for msg in messages: + if not isinstance(msg, dict) or msg.get("role") != "tool": + continue + call_id = msg.get("tool_call_id") + for content in msg.get("content", []) or []: + if not isinstance(content, dict) or content.get("type") != "tool_result": + continue + status = content.get("status") + if isinstance(status, str) and status in _FAILED_RUNTIME_STATUSES and call_id is not None: + failed_ids.append(call_id) + + ordered = [] + seen = set() + for call_id in failed_ids: + label = id_to_name.get(call_id) or call_id + if label and label not in seen: + seen.add(label) + ordered.append(label) + for name in failed_names_without_id: + if name and name not in seen: + seen.add(name) + ordered.append(name) + return ordered + + +def _get_tool_calls_results(agent_response_msgs): + """Extract formatted agent tool calls and results from a response. + + The output uses the ``[TOOL_CALL]`` / ``[TOOL_RESULT]`` line format. Tool results are rendered + via :func:`_stringify_tool_result` so list/dict grounding outputs are readable JSON. + + :param agent_response_msgs: The agent response messages to scan. + :type agent_response_msgs: List[dict] + :return: A list of formatted tool-call/result lines. + :rtype: List[str] + """ + agent_response_text = [] + tool_results = {} + + for msg in agent_response_msgs: + if msg.get("role") == "tool" and "tool_call_id" in msg: + for content in msg.get("content", []): + if content.get("type") == "tool_result": + result = content.get("tool_result") + tool_results[msg["tool_call_id"]] = f"[TOOL_RESULT] {_stringify_tool_result(result)}" + + for msg in agent_response_msgs: + if "role" in msg and msg.get("role") == "assistant" and "content" in msg: + for content in msg.get("content", []): + if content.get("type") == "tool_call": + if "tool_call" in content and "function" in content.get("tool_call", {}): + tc = content.get("tool_call", {}) + func_name = tc.get("function", {}).get("name", "") + args = tc.get("function", {}).get("arguments", {}) + tool_call_id = tc.get("id") + else: + tool_call_id = content.get("tool_call_id") + func_name = content.get("name", "") + args = content.get("arguments", {}) + args_str = ", ".join(f"{k}={_format_value(v)}" for k, v in args.items()) + call_line = f"[TOOL_CALL] {func_name}({args_str})" + agent_response_text.append(call_line) + if tool_call_id in tool_results: + agent_response_text.append(tool_results[tool_call_id]) + + return agent_response_text + + +def _reformat_tool_calls_results(response, logger=None): + """Reformat an agent response into tool-call/result lines, with a safe fallback. + + :param response: The agent response to reformat. + :type response: Union[None, List[dict], str] + :param logger: Optional logger for warning messages. + :type logger: Optional[logging.Logger] + :return: The formatted string, or the original response if parsing fails. + :rtype: Union[str, List[dict]] + """ + try: + if response is None or response == []: + return "" + agent_response = _get_tool_calls_results(response) + if agent_response == []: + if logger: + logger.warning( + "Empty agent response extracted, likely due to input schema change. " + "Falling back to using the original response. %s", + _log_safe_summary(response), + ) + return response + return "\n".join(agent_response) + except Exception as e: # pylint: disable=broad-except + if logger: + logger.warning( + "Agent response could not be parsed, falling back to original response. Error: %s. %s", + e, + _log_safe_summary(response), + ) + return response + + +def _get_conversation_history_with_tool_calls(query, include_system_messages=False): + """Parse conversation history, rendering tool calls/results inline within agent turns. + + This is the ``include_tool_calls=True`` variant used by tool-focused evaluators. Unlike the + default path, tool calls and their matched results are flattened into ``[TOOL_CALL]`` / + ``[TOOL_RESULT]`` lines within each agent turn. + + :param query: The list of conversation messages. + :type query: List[dict] + :param include_system_messages: Whether to capture the system message. + :type include_system_messages: bool + :return: Dict with ``user_queries``, ``agent_responses`` and optionally ``system_message``. + :rtype: Dict + :raises EvaluationException: If the conversation history is malformed. + """ + all_user_queries = [] + cur_user_query = [] + all_agent_responses = [] + cur_agent_response = [] + system_message = None + + tool_results = {} + for msg in query: + if msg.get("role") == "tool" and "tool_call_id" in msg: + tool_call_id = msg["tool_call_id"] + for content in msg.get("content", []): + if content.get("type") == "tool_result": + result = content.get("tool_result") + tool_results[tool_call_id] = f"[TOOL_RESULT] {_stringify_tool_result(result)}" + + for msg in query: + if "role" not in msg: + continue + + if include_system_messages and msg["role"] == "system" and "content" in msg: + system_message = msg.get("content", "") + + if msg["role"] == "user" and "content" in msg: + if cur_agent_response != []: + all_agent_responses.append(cur_agent_response) + cur_agent_response = [] + text_in_msg = _extract_text_from_content(msg["content"]) + if text_in_msg: + cur_user_query.append(text_in_msg) + + if msg["role"] == "assistant" and "content" in msg: + if cur_user_query != []: + all_user_queries.append(cur_user_query) + cur_user_query = [] + + text_in_msg = _extract_text_from_content(msg["content"]) + if text_in_msg: + cur_agent_response.append(text_in_msg) + + for content in msg.get("content", []): + if content.get("type") == "tool_call": + tool_call_id = content.get("tool_call_id") + func_name = content.get("name", "") + args = content.get("arguments", {}) + if "tool_call" in content and "function" in content.get("tool_call", {}): + tc = content.get("tool_call", {}) + func_name = tc.get("function", {}).get("name", "") + args = tc.get("function", {}).get("arguments", {}) + tool_call_id = tc.get("id") + args_str = ", ".join(f"{k}={_format_value(v)}" for k, v in args.items()) + tool_call_text = f"[TOOL_CALL] {func_name}({args_str})" + cur_agent_response.append(tool_call_text) + if tool_call_id and tool_call_id in tool_results: + cur_agent_response.append(tool_results[tool_call_id]) + + if cur_user_query != []: + all_user_queries.append(cur_user_query) + if cur_agent_response != []: + all_agent_responses.append(cur_agent_response) + + if len(all_user_queries) != len(all_agent_responses) + 1: + raise EvaluationException( + message=ErrorMessage.MALFORMED_CONVERSATION_HISTORY, + internal_message=ErrorMessage.MALFORMED_CONVERSATION_HISTORY, + target=ErrorTarget.CONVERSATION_HISTORY_PARSING, + category=ErrorCategory.INVALID_VALUE, + blame=ErrorBlame.USER_ERROR, + ) + + result = {"user_queries": all_user_queries, "agent_responses": all_agent_responses} + if include_system_messages: + result["system_message"] = system_message + return result + + def upload(path: str, container_client: ContainerClient, logger=None): """Upload files or directories to Azure Blob Storage using a container client. diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py index ad64739b0bc0..da99eb2922a9 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py @@ -6,7 +6,6 @@ import math import re import os -from itertools import chain from typing import Dict, Optional, TypeVar, Union, List if os.getenv("AI_EVALS_USE_PF_PROMPTY", "false").lower() == "true": @@ -76,20 +75,29 @@ def _drop_mcp_approval_messages(messages): def _normalize_function_call_types(messages): - """Normalize function_call/function_call_output types to tool_call/tool_result.""" + """Normalize function_call/function_call_output/openapi_call/openapi_call_output types to tool_call/tool_result.""" if not isinstance(messages, list): return messages for msg in messages: if isinstance(msg, dict) and isinstance(msg.get("content"), list): for item in msg["content"]: - if isinstance(item, dict) and item.get("type") == "function_call": + if not isinstance(item, dict): + continue + item_type = item.get("type") + if item_type == "function_call": item["type"] = "tool_call" if "function_call" in item: item["tool_call"] = item.pop("function_call") - elif isinstance(item, dict) and item.get("type") == "function_call_output": + elif item_type == "function_call_output": item["type"] = "tool_result" if "function_call_output" in item: item["tool_result"] = item.pop("function_call_output") + elif item_type == "openapi_call": + item["type"] = "tool_call" + elif item_type == "openapi_call_output": + item["type"] = "tool_result" + if "openapi_call_output" in item: + item["tool_result"] = item.pop("openapi_call_output") return messages @@ -370,13 +378,18 @@ def _extract_needed_tool_definitions( built_in_definitions = self._get_needed_built_in_tool_definitions(tool_calls) needed_tool_definitions.extend(built_in_definitions) - # OpenAPI tool is a collection of functions, so we need to expand it - tool_definitions_expanded = list( - chain.from_iterable( - tool.get("functions", []) if tool.get("type") == "openapi" else [tool] - for tool in needed_tool_definitions - ) - ) + # An OpenAPI tool is a collection of functions. When an OpenAPI tool + # definition exposes its nested functions, expand it into those functions + # so tool calls referencing a nested function name can be matched. If an + # OpenAPI tool definition has no nested functions, keep the OpenAPI tool + # definition as is for backward compatibility. + tool_definitions_expanded = [] + for tool in needed_tool_definitions: + openapi_functions = tool.get("functions") if tool.get("type") == "openapi" else None + if openapi_functions: + tool_definitions_expanded.extend(openapi_functions) + else: + tool_definitions_expanded.append(tool) # Validate that all tool calls have corresponding definitions for tool_call in tool_calls: diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py index 841c2e0b866a..8650416068ca 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py @@ -399,7 +399,7 @@ def _get_context_from_agent_response(self, response, tool_definitions): try: logger.debug("Extracting context from response") tool_calls = self._parse_tools_from_response(response=response) - logger.debug(f"Tool Calls parsed successfully: {tool_calls}") + logger.debug("Tool calls parsed successfully: count=%d", len(tool_calls) if tool_calls else 0) if not tool_calls: return NO_CONTEXT diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py index 44e0876bad68..1b639e9e36c4 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py @@ -14,6 +14,7 @@ ) from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase from azure.ai.evaluation._evaluators._common._validators import ToolDefinitionsValidator, ValidatorInterface +from azure.ai.evaluation._common.utils import _log_safe_summary, _stringify_tool_result from azure.ai.evaluation._common._experimental import experimental logger = logging.getLogger(__name__) @@ -283,7 +284,7 @@ def _get_tool_calls_results(agent_response_msgs): for content in msg.get("content", []): if content.get("type") == "tool_result": result = content.get("tool_result") - tool_results[msg["tool_call_id"]] = f"[TOOL_RESULT] {result}" + tool_results[msg["tool_call_id"]] = f"[TOOL_RESULT] {_stringify_tool_result(result)}" # Second pass: parse assistant messages and tool calls for msg in agent_response_msgs: @@ -320,18 +321,23 @@ def _reformat_tool_calls_results(response, logger=None): # fallback to the original response in that case if logger: logger.warning( - f"Empty agent response extracted, likely due to input schema change. " - f"Falling back to using the original response" + "Empty agent response extracted, likely due to input schema change. " + "Falling back to using the original response. %s", + _log_safe_summary(response), ) return response return "\n".join(agent_response) - except Exception: + except Exception as e: # pylint: disable=broad-except # If the agent response cannot be parsed for whatever # reason (e.g. the converter format changed), the original response is returned # This is a fallback to ensure that the evaluation can still proceed. # See comments on reformat_conversation_history for more details. if logger: - logger.warning(f"Agent response could not be parsed, falling back to original response") + logger.warning( + "Agent response could not be parsed, falling back to original response. Error: %s. %s", + e, + _log_safe_summary(response), + ) return response @@ -345,12 +351,14 @@ def _reformat_tool_definitions(tool_definitions, logger=None): param_names = ", ".join(params.keys()) if params else "no parameters" output_lines.append(f"- {name}: {desc} (inputs: {param_names})") return "\n".join(output_lines) - except Exception: + except Exception as e: # pylint: disable=broad-except # If the tool definitions cannot be parsed for whatever reason, the original tool definitions are returned # This is a fallback to ensure that the evaluation can still proceed. # See comments on reformat_conversation_history for more details. if logger: logger.warning( - f"Tool definitions could not be parsed, falling back to original definitions: {tool_definitions}" + "Tool definitions could not be parsed; falling back to raw definitions. Input shape: %s. Error: %s", + _log_safe_summary(tool_definitions), + e, ) return tool_definitions diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_exceptions.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_exceptions.py index 39227adc204d..c73dc13bb15a 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_exceptions.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_exceptions.py @@ -107,6 +107,10 @@ class ErrorTarget(Enum): QA_EVALUATOR = "QAEvaluator" ROUGE_EVALUATOR = "RougeScoreEvaluator" DOCUMENT_RETRIEVAL_EVALUATOR = "DocumentRetrievalEvaluator" + QUALITY_GRADER_EVALUATOR = "QualityGraderEvaluator" + CUSTOMER_SATISFACTION_EVALUATOR = "CustomerSatisfactionEvaluator" + DEFLECTION_RATE_EVALUATOR = "DeflectionRateEvaluator" + REGEX_MATCH_EVALUATOR = "RegexMatchEvaluator" class EvaluationException(AzureError): diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_multi_turn_utilities.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_multi_turn_utilities.py index bf2eca9587cf..9d4d664649c3 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_multi_turn_utilities.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_multi_turn_utilities.py @@ -804,5 +804,22 @@ def test_drops_mcp_and_normalizes(self): # function_call_output should be normalized to tool_result assert result[2]["content"][0]["type"] == "tool_result" + def test_normalizes_openapi_types(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "Call the API"}]}, + { + "role": "assistant", + "content": [{"type": "openapi_call", "name": "api", "arguments": {}, "tool_call_id": "c1"}], + }, + { + "role": "tool", + "content": [{"type": "openapi_call_output", "openapi_call_output": "api result"}], + }, + ] + result = _preprocess_messages(messages) + assert result[1]["content"][0]["type"] == "tool_call" + assert result[2]["content"][0]["type"] == "tool_result" + assert result[2]["content"][0]["tool_result"] == "api result" + # endregion diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_accuracy_evaluator.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_accuracy_evaluator.py index f02d589bbf58..3d8e1a960858 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_accuracy_evaluator.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_accuracy_evaluator.py @@ -2,7 +2,7 @@ import pytest from azure.ai.evaluation import ToolCallAccuracyEvaluator -from azure.ai.evaluation._exceptions import EvaluationException +from azure.ai.evaluation._exceptions import EvaluationException, ErrorTarget # This mock should return a dictionary that mimics the output of the prompty (the _flow call), @@ -733,6 +733,139 @@ def test_evaluate_open_api_with_tool_definition(self, mock_model_config): assert result[f"{key}_score"] == 5.0 assert result[f"{key}_passed"] is True + def test_extract_needed_tool_definitions_openapi_with_functions(self, mock_model_config): + """OpenAPI tool definitions that expose nested functions should be expanded so a + tool call referencing a nested function name is validated successfully.""" + evaluator = ToolCallAccuracyEvaluator(model_config=mock_model_config) + + tool_calls = [ + { + "type": "tool_call", + "tool_call_id": "call_1", + "name": "get_countries_LookupCountryByCurrency", + "arguments": {"currency": "GBP"}, + } + ] + tool_definitions = [ + { + "name": "get_countries", + "type": "openapi", + "description": "Retrieve a list of countries", + "functions": [ + { + "name": "get_countries_LookupCountryByCurrency", + "type": "function", + "description": "Search by currency.", + "parameters": { + "type": "object", + "properties": { + "currency": {"type": "string", "description": "The currency to search for."} + }, + "required": ["currency"], + }, + } + ], + } + ] + + needed_tool_definitions = evaluator._extract_needed_tool_definitions( + tool_calls, tool_definitions, ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR + ) + + # The nested function name resolved during validation (no exception was raised), + # and the original OpenAPI tool definition is returned unchanged. + assert needed_tool_definitions == tool_definitions + + def test_extract_needed_tool_definitions_openapi_without_functions(self, mock_model_config): + """OpenAPI tool definitions without nested functions should be kept as is so a + tool call referencing the top-level OpenAPI tool name is validated successfully.""" + evaluator = ToolCallAccuracyEvaluator(model_config=mock_model_config) + + tool_calls = [ + { + "type": "tool_call", + "tool_call_id": "call_1", + "name": "get_countries", + "arguments": {"currency": "GBP"}, + } + ] + tool_definitions = [ + { + "name": "get_countries", + "type": "openapi", + "description": "Retrieve a list of countries", + } + ] + + needed_tool_definitions = evaluator._extract_needed_tool_definitions( + tool_calls, tool_definitions, ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR + ) + + # The top-level OpenAPI tool name resolved during validation (no exception was raised). + assert needed_tool_definitions == tool_definitions + + def test_extract_needed_tool_definitions_openapi_empty_functions(self, mock_model_config): + """OpenAPI tool definitions with an empty functions list should fall back to the + top-level OpenAPI tool definition for validation.""" + evaluator = ToolCallAccuracyEvaluator(model_config=mock_model_config) + + tool_calls = [ + { + "type": "tool_call", + "tool_call_id": "call_1", + "name": "get_countries", + "arguments": {"currency": "GBP"}, + } + ] + tool_definitions = [ + { + "name": "get_countries", + "type": "openapi", + "description": "Retrieve a list of countries", + "functions": [], + } + ] + + needed_tool_definitions = evaluator._extract_needed_tool_definitions( + tool_calls, tool_definitions, ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR + ) + + assert needed_tool_definitions == tool_definitions + + def test_extract_needed_tool_definitions_openapi_missing_nested_function_raises(self, mock_model_config): + """A tool call referencing a nested function not present in an expanded OpenAPI tool + definition should still raise an EvaluationException.""" + evaluator = ToolCallAccuracyEvaluator(model_config=mock_model_config) + + tool_calls = [ + { + "type": "tool_call", + "tool_call_id": "call_1", + "name": "get_countries_UnknownFunction", + "arguments": {"currency": "GBP"}, + } + ] + tool_definitions = [ + { + "name": "get_countries", + "type": "openapi", + "functions": [ + { + "name": "get_countries_LookupCountryByCurrency", + "type": "function", + "parameters": {"type": "object", "properties": {}}, + } + ], + } + ] + + with pytest.raises(EvaluationException) as exc_info: + evaluator._extract_needed_tool_definitions( + tool_calls, tool_definitions, ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR + ) + + assert "Tool definition for get_countries_UnknownFunction not found" in str(exc_info.value) + def test_evaluate_missing_query(self, mock_model_config): """Test that evaluator raises exception when query is None or missing.""" evaluator = ToolCallAccuracyEvaluator(model_config=mock_model_config) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_logging.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_logging.py new file mode 100644 index 000000000000..5294ea518047 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_logging.py @@ -0,0 +1,62 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +import logging + +import pytest + +from azure.ai.evaluation._evaluators._tool_call_success._tool_call_success import ( + _reformat_tool_definitions, + _reformat_tool_calls_results, + _get_tool_calls_results, +) + +_SECRET = "SECRET_DO_NOT_LOG" + + +@pytest.mark.unittest +class TestToolCallSuccessSafeLogging: + """The fallback/debug paths must not log raw customer payloads (parity with azureml-assets #5158).""" + + def test_reformat_tool_definitions_fallback_does_not_leak_payload(self, caplog): + # A tool whose "parameters" is a string makes ``.get("properties")`` raise, triggering the fallback. + tool_definitions = [{"name": "t", "parameters": _SECRET}, _SECRET] + logger = logging.getLogger("test.tcs.tooldefs") + + with caplog.at_level(logging.WARNING): + result = _reformat_tool_definitions(tool_definitions, logger=logger) + + # Fallback returns the original definitions unchanged. + assert result == tool_definitions + # The raw payload must not appear in any log message; a structural summary is logged instead. + assert _SECRET not in caplog.text + assert "type=list" in caplog.text + + def test_reformat_tool_calls_results_fallback_does_not_leak_payload(self, caplog): + # A non-dict message makes ``msg.get(...)`` raise, triggering the fallback. + response = [_SECRET] + logger = logging.getLogger("test.tcs.results") + + with caplog.at_level(logging.WARNING): + result = _reformat_tool_calls_results(response, logger=logger) + + assert result == response + assert _SECRET not in caplog.text + assert "type=list" in caplog.text + + def test_get_tool_calls_results_serializes_dict_tool_result(self): + # Structured grounding-tool outputs (dict/list) must render as JSON, not a Python repr. + msgs = [ + { + "role": "assistant", + "content": [{"type": "tool_call", "name": "search", "arguments": {"q": "x"}, "tool_call_id": "c1"}], + }, + { + "role": "tool", + "tool_call_id": "c1", + "content": [{"type": "tool_result", "tool_result": {"docs": ["a", "b"]}}], + }, + ] + result = _get_tool_calls_results(msgs) + assert '[TOOL_RESULT] {"docs": ["a", "b"]}' in result diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_utils.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_utils.py index c3fda19e794d..91bbeee07a1b 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_utils.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_utils.py @@ -16,8 +16,15 @@ reformat_agent_response, reformat_tool_definitions, construct_prompty_model_config, + _stringify_tool_result, + _log_safe_summary, + _coerce_bool, + _coerce_number, + _collect_failed_tool_calls, + _get_tool_calls_results, + _reformat_tool_calls_results, ) -from azure.ai.evaluation._exceptions import EvaluationException, ErrorMessage +from azure.ai.evaluation._exceptions import EvaluationException, ErrorMessage, ErrorTarget @pytest.mark.unittest @@ -984,3 +991,243 @@ def test_extra_headers_invalid_type_raises_error(self): } with pytest.raises(EvaluationException): construct_prompty_model_config(model_config, "2024-02-01", "") + + +@pytest.mark.unittest +class TestToolResultAndLoggingHelpers: + """Tests for helpers added for parity with azureml-assets builtin evaluators.""" + + # region _stringify_tool_result + + def test_stringify_tool_result_none_returns_empty(self): + assert _stringify_tool_result(None) == "" + + def test_stringify_tool_result_string_passthrough(self): + assert _stringify_tool_result("plain text") == "plain text" + + def test_stringify_tool_result_dict_is_json(self): + result = _stringify_tool_result({"title": "Doc", "score": 1}) + # Valid JSON with double quotes (not a Python repr with single quotes) + assert json.loads(result) == {"title": "Doc", "score": 1} + assert "'" not in result + + def test_stringify_tool_result_list_is_json(self): + result = _stringify_tool_result([{"url": "https://x"}, {"url": "https://y"}]) + assert json.loads(result) == [{"url": "https://x"}, {"url": "https://y"}] + + def test_stringify_tool_result_non_serializable_falls_back_to_str(self): + class _Weird: + def __str__(self): + return "weird-value" + + # default=str handles non-JSON-native values without raising + assert _stringify_tool_result({"k": _Weird()}) == '{"k": "weird-value"}' + + # endregion + + # region _get_agent_response uses _stringify_tool_result + + def test_get_agent_response_serializes_dict_tool_result(self): + agent_msgs = [ + { + "role": "assistant", + "content": [ + {"type": "tool_call", "name": "search", "arguments": {"q": "hi"}, "tool_call_id": "c1"}, + ], + }, + { + "role": "tool", + "tool_call_id": "c1", + "content": [{"type": "tool_result", "tool_result": {"docs": ["a", "b"]}}], + }, + ] + result = _get_agent_response(agent_msgs, include_tool_messages=True) + # tool result rendered as JSON, not python repr + assert '[TOOL_RESULT] {"docs": ["a", "b"]}' in result + + # endregion + + # region _log_safe_summary + + def test_log_safe_summary_list_reports_roles(self): + summary = _log_safe_summary([{"role": "user"}, {"role": "assistant"}]) + assert "type=list" in summary + assert "len=2" in summary + assert "user" in summary and "assistant" in summary + + def test_log_safe_summary_dict_reports_keys_not_values(self): + summary = _log_safe_summary({"secret": "s3cr3t", "api_key": "abc"}) + assert "type=dict" in summary + assert "api_key" in summary and "secret" in summary + # values must never leak + assert "s3cr3t" not in summary and "abc" not in summary + + def test_log_safe_summary_scalar(self): + assert "type=int" in _log_safe_summary(42) + + # endregion + + # region _coerce_bool / _coerce_number + + def test_coerce_bool(self): + assert _coerce_bool(True) is True + assert _coerce_bool(False) is False + assert _coerce_bool("true") is True + assert _coerce_bool(" FALSE ") is False + assert _coerce_bool("nonsense") is None + assert _coerce_bool(None) is None + assert _coerce_bool(1) is None + + def test_coerce_number(self): + assert _coerce_number(3) == 3 + assert _coerce_number(2.5) == 2.5 + assert _coerce_number("2.5") == 2.5 + assert _coerce_number("null") is None + assert _coerce_number("") is None + assert _coerce_number(True) is None # bools are not numbers here + assert _coerce_number("abc") is None + + # endregion + + # region _collect_failed_tool_calls + + def test_collect_failed_tool_calls_by_status_on_tool_call(self): + messages = [ + { + "role": "assistant", + "content": [ + {"type": "tool_call", "name": "ok_tool", "tool_call_id": "c1", "status": "completed"}, + {"type": "tool_call", "name": "bad_tool", "tool_call_id": "c2", "status": "failed"}, + ], + } + ] + assert _collect_failed_tool_calls(messages) == ["bad_tool"] + + def test_collect_failed_tool_calls_by_status_on_tool_result(self): + messages = [ + { + "role": "assistant", + "content": [{"type": "tool_call", "name": "search", "tool_call_id": "c1"}], + }, + { + "role": "tool", + "tool_call_id": "c1", + "content": [{"type": "tool_result", "tool_result": "x", "status": "incomplete"}], + }, + ] + assert _collect_failed_tool_calls(messages) == ["search"] + + def test_collect_failed_tool_calls_none_failed(self): + messages = [ + {"role": "assistant", "content": [{"type": "tool_call", "name": "search", "tool_call_id": "c1"}]}, + ] + assert _collect_failed_tool_calls(messages) == [] + + def test_collect_failed_tool_calls_non_list(self): + assert _collect_failed_tool_calls("not a list") == [] + + # endregion + + # region _get_tool_calls_results / _reformat_tool_calls_results + + def test_get_tool_calls_results_formats_calls_and_results(self): + msgs = [ + { + "role": "assistant", + "content": [{"type": "tool_call", "name": "search", "arguments": {"q": "cats"}, "tool_call_id": "c1"}], + }, + { + "role": "tool", + "tool_call_id": "c1", + "content": [{"type": "tool_result", "tool_result": {"n": 3}}], + }, + ] + result = _get_tool_calls_results(msgs) + assert result == ['[TOOL_CALL] search(q="cats")', '[TOOL_RESULT] {"n": 3}'] + + def test_reformat_tool_calls_results_empty(self): + assert _reformat_tool_calls_results([]) == "" + assert _reformat_tool_calls_results(None) == "" + + def test_reformat_tool_calls_results_joins_lines(self): + msgs = [ + { + "role": "assistant", + "content": [{"type": "tool_call", "name": "t", "arguments": {}, "tool_call_id": "c"}], + }, + {"role": "tool", "tool_call_id": "c", "content": [{"type": "tool_result", "tool_result": "done"}]}, + ] + result = _reformat_tool_calls_results(msgs) + assert result == "[TOOL_CALL] t()\n[TOOL_RESULT] done" + + # endregion + + # region _get_conversation_history include_tool_calls + + def test_get_conversation_history_with_tool_calls(self): + query = [ + {"role": "user", "content": [{"type": "text", "text": "Find hotels"}]}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Searching"}, + {"type": "tool_call", "name": "search", "arguments": {"loc": "Paris"}, "tool_call_id": "c1"}, + ], + }, + { + "role": "tool", + "tool_call_id": "c1", + "content": [{"type": "tool_result", "tool_result": "Found 3"}], + }, + {"role": "user", "content": [{"type": "text", "text": "thanks"}]}, + ] + result = _get_conversation_history(query, include_tool_calls=True) + assert result["user_queries"] == [[["Find hotels"]], [["thanks"]]] + # Agent turn contains inline tool call + result lines + agent_turn = result["agent_responses"][0] + assert ["Searching"] in agent_turn + assert '[TOOL_CALL] search(loc="Paris")' in agent_turn + assert "[TOOL_RESULT] Found 3" in agent_turn + + def test_reformat_conversation_history_include_tool_calls_renders_inline(self): + query = [ + {"role": "user", "content": [{"type": "text", "text": "Find hotels"}]}, + { + "role": "assistant", + "content": [ + {"type": "tool_call", "name": "search", "arguments": {"loc": "Paris"}, "tool_call_id": "c1"}, + {"type": "text", "text": "Found some"}, + ], + }, + {"role": "user", "content": [{"type": "text", "text": "thanks"}]}, + ] + result = reformat_conversation_history(query, include_tool_calls=True) + assert '[TOOL_CALL] search(loc="Paris")' in result + + def test_get_conversation_history_default_path_unchanged(self): + """Default (include_tool_calls=False) behavior must be unchanged.""" + query = [ + {"role": "user", "content": [{"type": "text", "text": "hi"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "hello"}]}, + {"role": "user", "content": [{"type": "text", "text": "bye"}]}, + ] + result = _get_conversation_history(query) + assert result == { + "user_queries": [[["hi"]], [["bye"]]], + "agent_responses": [[["hello"]]], + } + + # endregion + + # region ErrorTarget parity members + + def test_error_target_parity_members_exist(self): + for name in ( + "QUALITY_GRADER_EVALUATOR", + "CUSTOMER_SATISFACTION_EVALUATOR", + "DEFLECTION_RATE_EVALUATOR", + "REGEX_MATCH_EVALUATOR", + ): + assert hasattr(ErrorTarget, name) + + # endregion