Skip to content

Python: Keep span-only tool metadata off invocation metrics #9567

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Alexander Kaplan (tokaplan) wants to merge 1 commit into from
Closed

Python: Keep span-only tool metadata off invocation metrics #9567

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions python/packages/core/agent_framework/_tools.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1231,8 +1231,12 @@ async def invoke(
return parsed

span_attributes = get_function_span_attributes(self, tool_call_id=tool_call_id)
metric_attributes = dict(span_attributes)
metric_attributes[OtelAttr.MEASUREMENT_FUNCTION_TAG_NAME] = self.name
metric_attributes = {
OtelAttr.OPERATION: span_attributes[OtelAttr.OPERATION],
OtelAttr.TOOL_NAME: span_attributes[OtelAttr.TOOL_NAME],
OtelAttr.TOOL_TYPE: span_attributes[OtelAttr.TOOL_TYPE],
OtelAttr.MEASUREMENT_FUNCTION_TAG_NAME: self.name,
}
# Filter out framework kwargs that are not JSON serializable.
serializable_kwargs = {
k: v
Expand Down
32 changes: 26 additions & 6 deletions python/packages/core/tests/core/test_tools.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -999,9 +999,11 @@ def multiply(self, factor: int) -> str:
async def test_tool_invoke_telemetry_enabled(span_exporter: InMemorySpanExporter):
"""Test the tool invoke method with telemetry enabled."""

tool_description = "Sensitive telemetry tool description"

@tool(
name="telemetry_test_tool",
description="A test tool for telemetry",
description=tool_description,
)
def telemetry_test_tool(expression: str) -> str:
"""A function that echoes an expression for telemetry testing."""
Expand All @@ -1028,7 +1030,7 @@ def telemetry_test_tool(expression: str) -> str:
assert span.attributes[OtelAttr.TOOL_NAME] == "telemetry_test_tool" # type: ignore[index] # pyrefly: ignore[unsupported-operation] # ty: ignore[not-subscriptable]
assert span.attributes[OtelAttr.TOOL_CALL_ID] == "test_call_id" # type: ignore[index] # pyrefly: ignore[unsupported-operation] # ty: ignore[not-subscriptable]
assert span.attributes[OtelAttr.TOOL_TYPE] == "function" # type: ignore[index] # pyrefly: ignore[unsupported-operation] # ty: ignore[not-subscriptable]
assert span.attributes[OtelAttr.TOOL_DESCRIPTION] == "A test tool for telemetry" # type: ignore[index] # pyrefly: ignore[unsupported-operation] # ty: ignore[not-subscriptable]
assert span.attributes[OtelAttr.TOOL_DESCRIPTION] == tool_description # type: ignore[index] # pyrefly: ignore[unsupported-operation] # ty: ignore[not-subscriptable]
assert span.attributes[OtelAttr.TOOL_ARGUMENTS] == '{"expression": "(12345 + 67890) / 3"}' # type: ignore[index] # pyrefly: ignore[unsupported-operation] # ty: ignore[not-subscriptable]
assert span.attributes[OtelAttr.TOOL_RESULT] == expression # type: ignore[index] # pyrefly: ignore[unsupported-operation] # ty: ignore[not-subscriptable]

Expand All @@ -1037,11 +1039,16 @@ def telemetry_test_tool(expression: str) -> str:
call_args = mock_histogram.record.call_args
assert call_args[0][0] > 0 # duration should be positive
attributes = call_args[1]["attributes"]
assert attributes[OtelAttr.OPERATION] == OtelAttr.TOOL_EXECUTION_OPERATION
assert attributes[OtelAttr.TOOL_NAME] == "telemetry_test_tool"
assert attributes[OtelAttr.TOOL_TYPE] == "function"
assert attributes[OtelAttr.MEASUREMENT_FUNCTION_TAG_NAME] == "telemetry_test_tool"
assert attributes[OtelAttr.TOOL_CALL_ID] == "test_call_id"
assert OtelAttr.TOOL_CALL_ID not in attributes
assert OtelAttr.TOOL_DESCRIPTION not in attributes
assert OtelAttr.TOOL_ARGUMENTS not in attributes
assert OtelAttr.TOOL_RESULT not in attributes
assert expression not in attributes.values()
assert tool_description not in attributes.values()


@pytest.mark.parametrize("enable_sensitive_data", [False], indirect=True)
Expand Down Expand Up @@ -1085,8 +1092,12 @@ def telemetry_test_tool(x: int, y: int) -> int:
call_args = mock_histogram.record.call_args
assert call_args[0][0] > 0 # duration should be positive
attributes = call_args[1]["attributes"]
assert attributes[OtelAttr.OPERATION] == OtelAttr.TOOL_EXECUTION_OPERATION
assert attributes[OtelAttr.TOOL_NAME] == "telemetry_test_tool"
assert attributes[OtelAttr.TOOL_TYPE] == "function"
assert attributes[OtelAttr.MEASUREMENT_FUNCTION_TAG_NAME] == "telemetry_test_tool"
assert attributes[OtelAttr.TOOL_CALL_ID] == "test_call_id"
assert OtelAttr.TOOL_CALL_ID not in attributes
assert OtelAttr.TOOL_DESCRIPTION not in attributes


@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
Expand Down Expand Up @@ -1179,10 +1190,11 @@ async def test_tool_invoke_telemetry_with_exception(
) -> None:
"""Test the tool invoke method with telemetry when an exception occurs."""
private_error_detail = "founder-secret-token"
tool_description = "Sensitive exception tool description"

@tool(
name="exception_test_tool",
description="A test tool that raises an exception",
description=tool_description,
)
def exception_test_tool(x: int, y: int) -> int:
"""A function that raises an exception for telemetry testing."""
Expand All @@ -1209,7 +1221,7 @@ def exception_test_tool(x: int, y: int) -> int:
assert span.attributes[OtelAttr.TOOL_NAME] == "exception_test_tool" # type: ignore[index] # pyrefly: ignore[unsupported-operation] # ty: ignore[not-subscriptable]
assert span.attributes[OtelAttr.TOOL_CALL_ID] == "exception_call" # type: ignore[index] # pyrefly: ignore[unsupported-operation] # ty: ignore[not-subscriptable]
assert span.attributes[OtelAttr.TOOL_TYPE] == "function" # type: ignore[index] # pyrefly: ignore[unsupported-operation] # ty: ignore[not-subscriptable]
assert span.attributes[OtelAttr.TOOL_DESCRIPTION] == "A test tool that raises an exception" # type: ignore[index] # pyrefly: ignore[unsupported-operation] # ty: ignore[not-subscriptable]
assert span.attributes[OtelAttr.TOOL_DESCRIPTION] == tool_description # type: ignore[index] # pyrefly: ignore[unsupported-operation] # ty: ignore[not-subscriptable]
if enable_sensitive_data:
assert span.attributes[OtelAttr.TOOL_ARGUMENTS] == '{"x": 1, "y": 2}' # type: ignore[index] # pyrefly: ignore[unsupported-operation] # ty: ignore[not-subscriptable]
else:
Expand All @@ -1228,9 +1240,17 @@ def exception_test_tool(x: int, y: int) -> int:
mock_histogram.record.assert_called_once()
call_args = mock_histogram.record.call_args
attributes = call_args[1]["attributes"]
assert attributes[OtelAttr.OPERATION] == OtelAttr.TOOL_EXECUTION_OPERATION
assert attributes[OtelAttr.TOOL_NAME] == "exception_test_tool"
assert attributes[OtelAttr.TOOL_TYPE] == "function"
assert attributes[OtelAttr.MEASUREMENT_FUNCTION_TAG_NAME] == "exception_test_tool"
assert attributes[OtelAttr.ERROR_TYPE] == ValueError.__name__
assert OtelAttr.TOOL_CALL_ID not in attributes
assert OtelAttr.TOOL_DESCRIPTION not in attributes
assert OtelAttr.TOOL_ARGUMENTS not in attributes
assert OtelAttr.TOOL_RESULT not in attributes
assert tool_description not in attributes.values()
assert private_error_detail not in attributes.values()


async def test_tool_invoke_telemetry_async_function(span_exporter: InMemorySpanExporter):
Expand Down
Loading