Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions python/packages/core/agent_framework/_evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -1064,12 +1064,22 @@ def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") ->
Returns:
A check function for use with ``LocalEvaluator``.

Raises:
ValueError: If no tool names are provided, if any tool name is not a non-empty string,
or if mode is not 'all' or 'any'.

Example:

.. code-block:: python

check = tool_called_check("get_weather", "get_flight_price")
"""
if not tool_names:
raise ValueError("At least one tool name must be provided to tool_called_check.")
if any(not isinstance(t, str) or not t.strip() for t in tool_names):
raise ValueError("Tool names must be non-empty strings.")
if mode not in ("all", "any"):
raise ValueError(f"Invalid mode: {mode!r}. Must be 'all' or 'any'.")

def _check(item: EvalItem) -> CheckResult:
expected = set(tool_names)
Expand Down
19 changes: 19 additions & 0 deletions python/packages/core/tests/core/test_local_eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -973,6 +973,25 @@ async def test_any_mode_none_called(self):
assert "None of expected tools" in result.reason # type: ignore[union-attr] # ty: ignore[unresolved-attribute]


class TestToolCalledCheckValidation:
"""Tests for tool_called_check argument validation."""

def test_tool_called_check_requires_at_least_one_tool(self):
with pytest.raises(ValueError, match="At least one tool name"):
tool_called_check()

def test_tool_called_check_rejects_empty_tool_name(self):
with pytest.raises(ValueError, match="non-empty strings"):
tool_called_check("valid_tool", "")

with pytest.raises(ValueError, match="non-empty strings"):
tool_called_check(" ")

def test_tool_called_check_rejects_invalid_mode(self):
with pytest.raises(ValueError, match="Invalid mode"):
tool_called_check("tool_a", mode="invalid") # type: ignore[arg-type]


class TestCoerceResultScoreError:
"""Tests for _coerce_result handling non-numeric score."""

Expand Down
Loading