Merge pull request #324 from scaleapi/dm/add-shell-tool-support

Add ShellTool support to TemporalStreamingModel

Author: danielmillerp

pyproject.toml

@@ -25,23 +25,23 @@ dependencies = [
     "pyyaml>=6.0.2,<7",
     "jsonschema>=4.23.0,<5",
     "jsonref>=1.1.0,<2",
-    "temporalio>=1.18.2,<2",
+    "temporalio>=1.26.0,<2",
     "aiohttp>=3.10.10,<4",
     "redis>=5.2.0,<6",
     "litellm>=1.83.0,<2",
     "kubernetes>=25.0.0,<36.0.0",
     "jinja2>=3.1.3,<4",
     "mcp[cli]>=1.4.1",
     "scale-gp>=0.1.0a59",
-    "openai-agents==0.4.2",
+    "openai-agents==0.14.1",
     "tzlocal>=5.3.1",
     "tzdata>=2025.2",
     "pytest>=8.4.0",
     "json_log_formatter>=1.1.1",
     "pytest-asyncio>=1.0.0",
     "scale-gp-beta>=0.1.0a20",
     "ipykernel>=6.29.5",
-    "openai>=2.2,<3", # Required by openai-agents 0.4.2; litellm now supports openai 2.x (issue #13711 resolved: https://github.com/BerriAI/litellm/issues/13711)
+    "openai>=2.2,<3", # Required by openai-agents; litellm now supports openai 2.x (issue #13711 resolved: https://github.com/BerriAI/litellm/issues/13711)
     "cloudpickle>=3.1.1",
     "datadog>=0.52.1",
     "ddtrace>=3.13.0",

requirements-dev.lock

@@ -64,8 +64,6 @@ click==8.3.1
     # via uvicorn
 cloudpickle==3.1.2
     # via agentex-sdk
-colorama==0.4.6
-    # via griffe
 colorlog==6.10.1
     # via nox
 comm==0.2.3
@@ -114,7 +112,7 @@ fsspec==2026.3.0
     # via huggingface-hub
 google-auth==2.49.1
     # via kubernetes
-griffe==1.15.0
+griffelib==2.0.2
     # via openai-agents
 h11==0.16.0
     # via httpcore
@@ -194,7 +192,7 @@ langgraph-checkpoint==4.0.1
     # via agentex-sdk
 langsmith==0.7.22
     # via langchain-core
-litellm==1.82.6
+litellm==1.83.0
     # via agentex-sdk
 markdown-it-py==3.0.0
     # via rich
@@ -229,7 +227,7 @@ openai==2.30.0
     # via agentex-sdk
     # via litellm
     # via openai-agents
-openai-agents==0.4.2
+openai-agents==0.14.1
     # via agentex-sdk
 opentelemetry-api==1.40.0
     # via agentex-sdk
@@ -391,7 +389,7 @@ stack-data==0.6.3
 starlette==0.46.2
     # via fastapi
     # via mcp
-temporalio==1.24.0
+temporalio==1.26.0
     # via agentex-sdk
 tenacity==9.1.4
     # via langchain-core
@@ -478,6 +476,8 @@ wcwidth==0.6.0
     # via prompt-toolkit
 websocket-client==1.9.0
     # via kubernetes
+websockets==15.0.1
+    # via openai-agents
 wrapt==2.1.2
     # via ddtrace
 xxhash==3.6.0

requirements.lock

@@ -61,8 +61,6 @@ click==8.3.1
     # via uvicorn
 cloudpickle==3.1.2
     # via agentex-sdk
-colorama==0.4.6
-    # via griffe
 comm==0.2.3
     # via ipykernel
 cryptography==46.0.6
@@ -101,7 +99,7 @@ fsspec==2026.3.0
     # via huggingface-hub
 google-auth==2.49.1
     # via kubernetes
-griffe==1.15.0
+griffelib==2.0.2
     # via openai-agents
 h11==0.16.0
     # via httpcore
@@ -178,7 +176,7 @@ langgraph-checkpoint==4.0.1
     # via agentex-sdk
 langsmith==0.7.22
     # via langchain-core
-litellm==1.82.6
+litellm==1.83.0
     # via agentex-sdk
 markdown-it-py==4.0.0
     # via rich
@@ -207,7 +205,7 @@ openai==2.30.0
     # via agentex-sdk
     # via litellm
     # via openai-agents
-openai-agents==0.4.2
+openai-agents==0.14.1
     # via agentex-sdk
 opentelemetry-api==1.40.0
     # via agentex-sdk
@@ -359,7 +357,7 @@ stack-data==0.6.3
 starlette==0.46.2
     # via fastapi
     # via mcp
-temporalio==1.24.0
+temporalio==1.26.0
     # via agentex-sdk
 tenacity==9.1.4
     # via langchain-core
@@ -441,6 +439,8 @@ wcwidth==0.6.0
     # via prompt-toolkit
 websocket-client==1.9.0
     # via kubernetes
+websockets==15.0.1
+    # via openai-agents
 wrapt==2.1.2
     # via ddtrace
 xxhash==3.6.0

src/agentex/lib/core/temporal/plugins/openai_agents/models/temporal_streaming_model.py

@@ -27,6 +27,12 @@
     CodeInterpreterTool,
     ImageGenerationTool,
 )
+from agents.computer import Computer, AsyncComputer
+
+try:
+    from agents.tool import ShellTool  # type: ignore[attr-defined]
+except ImportError:
+    ShellTool = None  # type: ignore[assignment,misc]
 from agents.usage import Usage, InputTokensDetails, OutputTokensDetails  # type: ignore[attr-defined]
 from agents.model_settings import MCPToolChoice
 from openai.types.responses import (
@@ -303,11 +309,28 @@ def _convert_tools(self, tools: list[Tool], handoffs: list[Handoff]) -> tuple[Li
                     tool_includes.append("file_search_call.results")
 
             elif isinstance(tool, ComputerTool):
+                # In newer openai-agents, tool.computer may be a factory
+                # (ComputerCreate/ComputerProvider). Only concrete Computer
+                # / AsyncComputer instances expose environment/dimensions.
+                computer = tool.computer
+                if not isinstance(computer, (Computer, AsyncComputer)):
+                    raise ValueError(
+                        "ComputerTool.computer must be a Computer or AsyncComputer "
+                        "instance for Responses API serialization; got "
+                        f"{type(computer).__name__}"
+                    )
+                environment = computer.environment
+                dimensions = computer.dimensions
+                if environment is None or dimensions is None:
+                    raise ValueError(
+                        "ComputerTool requires `environment` and `dimensions` on the "
+                        "Computer/AsyncComputer implementation."
+                    )
                 response_tools.append({
                     "type": "computer_use_preview",
-                    "environment": tool.computer.environment,
-                    "display_width": tool.computer.dimensions[0],
-                    "display_height": tool.computer.dimensions[1],
+                    "environment": environment,
+                    "display_width": dimensions[0],
+                    "display_height": dimensions[1],
                 })
 
             elif isinstance(tool, HostedMCPTool):
@@ -326,6 +349,13 @@ def _convert_tools(self, tools: list[Tool], handoffs: list[Handoff]) -> tuple[Li
                     "type": "local_shell",
                 })
 
+            elif ShellTool is not None and isinstance(tool, ShellTool):
+                environment = dict(tool.environment) if tool.environment else {"type": "local"}
+                response_tools.append({
+                    "type": "shell",
+                    "environment": environment,
+                })
+
             else:
                 logger.warning(f"Unknown tool type: {type(tool).__name__}, skipping")
 

src/agentex/lib/core/temporal/plugins/openai_agents/tests/test_convert_tools.py

@@ -0,0 +1,61 @@
+"""Unit tests for TemporalStreamingModel._convert_tools tool serialization."""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from agentex.lib.core.temporal.plugins.openai_agents.models import (
+    temporal_streaming_model as tsm_module,
+)
+from agentex.lib.core.temporal.plugins.openai_agents.models.temporal_streaming_model import (
+    TemporalStreamingModel,
+)
+
+
+@pytest.fixture
+def model():
+    with patch(
+        "agentex.lib.core.temporal.plugins.openai_agents.models.temporal_streaming_model.create_async_agentex_client"
+    ):
+        return TemporalStreamingModel(model_name="gpt-4o", openai_client=MagicMock())
+
+
+class _FakeShellTool:
+    """Stand-in for agents.tool.ShellTool for environments where it isn't installed."""
+
+    def __init__(self, environment):
+        self.environment = environment
+
+
+def test_shell_tool_local_environment(model, monkeypatch):
+    """ShellTool with a local environment should serialize to a 'shell' payload."""
+    monkeypatch.setattr(tsm_module, "ShellTool", _FakeShellTool)
+
+    tool = _FakeShellTool(environment={"type": "local", "skills": ["git"]})
+    response_tools, _ = model._convert_tools([tool], handoffs=[])
+
+    assert response_tools == [{"type": "shell", "environment": {"type": "local", "skills": ["git"]}}]
+
+
+def test_shell_tool_defaults_environment_when_missing(model, monkeypatch):
+    """ShellTool with environment=None should fall back to {'type': 'local'}."""
+    monkeypatch.setattr(tsm_module, "ShellTool", _FakeShellTool)
+
+    tool = _FakeShellTool(environment=None)
+    response_tools, _ = model._convert_tools([tool], handoffs=[])
+
+    assert response_tools == [{"type": "shell", "environment": {"type": "local"}}]
+
+
+def test_shell_tool_unavailable_falls_through(model, monkeypatch, caplog):
+    """If ShellTool isn't installed, an unknown tool should log a warning and be skipped."""
+    monkeypatch.setattr(tsm_module, "ShellTool", None)
+
+    class _NotAShellTool:
+        pass
+
+    with caplog.at_level("WARNING"):
+        response_tools, _ = model._convert_tools([_NotAShellTool()], handoffs=[])
+
+    assert response_tools == []
+    assert any("Unknown tool type" in rec.message for rec in caplog.records)