diff --git a/pyproject.toml b/pyproject.toml index dc93a7151..a85117b3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-langchain" -version = "0.14.17" +version = "0.14.18" description = "Python SDK that enables developers to build and deploy LangGraph agents to the UiPath Cloud Platform" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" @@ -26,7 +26,7 @@ dependencies = [ "pillow>=12.1.1", "rdflib>=7.0.0, <8.0.0", "a2a-sdk>=0.2.0,<1.0.0", - "uipath-langchain-client[openai]>=1.17.1,<1.18.0", + "uipath-langchain-client[openai]>=1.18.0,<1.19.0", ] classifiers = [ @@ -43,21 +43,21 @@ maintainers = [ [project.optional-dependencies] anthropic = [ - "uipath-langchain-client[anthropic]>=1.17.1,<1.18.0", + "uipath-langchain-client[anthropic]>=1.18.0,<1.19.0", ] vertex = [ - "uipath-langchain-client[google]>=1.17.1,<1.18.0", - "uipath-langchain-client[vertexai]>=1.17.1,<1.18.0", + "uipath-langchain-client[google]>=1.18.0,<1.19.0", + "uipath-langchain-client[vertexai]>=1.18.0,<1.19.0", ] bedrock = [ - "uipath-langchain-client[bedrock]>=1.17.1,<1.18.0", + "uipath-langchain-client[bedrock]>=1.18.0,<1.19.0", "boto3-stubs>=1.41.4", ] fireworks = [ - "uipath-langchain-client[fireworks]>=1.17.1,<1.18.0", + "uipath-langchain-client[fireworks]>=1.18.0,<1.19.0", ] all = [ - "uipath-langchain-client[all]>=1.17.1,<1.18.0", + "uipath-langchain-client[all]>=1.18.0,<1.19.0", ] [project.entry-points."uipath.middlewares"] diff --git a/src/uipath_langchain/agent/exceptions/licensing.py b/src/uipath_langchain/agent/exceptions/licensing.py deleted file mode 100644 index be338bc23..000000000 --- a/src/uipath_langchain/agent/exceptions/licensing.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Map a normalized LLM-client ``UiPathAPIError`` into an ``AgentRuntimeError``. - -The LLM client (uipath-llm-client / uipath-langchain-client) normalizes provider -HTTP errors into a ``UiPathAPIError`` carrying ``status_code`` and ``body``. This -module maps that status code to an ``AgentRuntimeError`` so upstream handling -(exception mapper, CAS bridge) can categorise without provider-specific logic. -""" - -from typing import NoReturn - -from uipath.llm_client import UiPathAPIError -from uipath.runtime.errors import UiPathErrorCategory - -from uipath_langchain.agent.exceptions.exceptions import ( - AgentRuntimeError, - AgentRuntimeErrorCode, -) - -# Maps known LLM Gateway status codes to specific error codes. -# Unknown status codes fall back to HTTP_ERROR. -_LLM_STATUS_CODE_MAP: dict[int, AgentRuntimeErrorCode] = { - 403: AgentRuntimeErrorCode.LICENSE_NOT_AVAILABLE, -} - - -def raise_for_provider_http_error(error: UiPathAPIError) -> NoReturn: - """Convert a normalized ``UiPathAPIError`` into a structured ``AgentRuntimeError``. - - Reads the HTTP status code and the gateway's ``detail`` (from ``error.body``) - and re-raises as an ``AgentRuntimeError`` chained on the original. - """ - status_code = error.status_code - code = _LLM_STATUS_CODE_MAP.get(status_code, AgentRuntimeErrorCode.HTTP_ERROR) - category = ( - UiPathErrorCategory.DEPLOYMENT - if status_code == 403 - else UiPathErrorCategory.UNKNOWN - ) - detail = error.body.get("detail") if isinstance(error.body, dict) else None - - raise AgentRuntimeError( - code=code, - title=f"LLM provider returned HTTP {status_code}", - detail=detail or error.message or str(error), - category=category, - status=status_code, - ) from error diff --git a/src/uipath_langchain/agent/exceptions/llm.py b/src/uipath_langchain/agent/exceptions/llm.py index 3a1c95453..30ea6320c 100644 --- a/src/uipath_langchain/agent/exceptions/llm.py +++ b/src/uipath_langchain/agent/exceptions/llm.py @@ -1,6 +1,16 @@ -"""Map normalized LLM-client errors into agent runtime errors.""" +"""Map normalized LLM-client errors into agent runtime errors. -from uipath.llm_client import UiPathError, UiPathLLMErrorCode +The LLM client (uipath-llm-client / uipath-langchain-client) surfaces two shapes: +a ``UiPathError`` carrying a semantic ``error_code`` (handled by +``raise_for_llm_client_error``), and a ``UiPathAPIError`` carrying an HTTP +``status_code`` + ``body`` for provider passthrough failures (handled by +``raise_for_provider_http_error``). Both are mapped to ``AgentRuntimeError`` so +upstream handling can categorise without provider-specific logic. +""" + +from typing import NoReturn + +from uipath.llm_client import UiPathAPIError, UiPathError, UiPathLLMErrorCode from uipath.runtime.errors import UiPathErrorCategory from uipath_langchain.agent.exceptions.exceptions import ( @@ -8,6 +18,12 @@ AgentRuntimeErrorCode, ) +# Maps known LLM Gateway status codes to specific error codes. +# Unknown status codes fall back to HTTP_ERROR. +_LLM_STATUS_CODE_MAP: dict[int, AgentRuntimeErrorCode] = { + 403: AgentRuntimeErrorCode.LICENSE_NOT_AVAILABLE, +} + def raise_for_llm_client_error(error: UiPathError) -> None: """Raise a structured agent error for known LLM-client error codes.""" @@ -22,3 +38,58 @@ def raise_for_llm_client_error(error: UiPathError) -> None: ), category=UiPathErrorCategory.USER, ) from error + + +def _extract_provider_detail(body: object) -> str | None: + """Pull the human-readable message out of an error response body. + + Tries the gateway's own envelope (``{"detail": ...}``) first, then the + vendor envelopes forwarded on passthrough 4xx responses (OpenAI/Anthropic + ``{"error": {"message": ...}}``, Vertex list-wrapped variants, flat + ``{"message": ...}``), then falls back to a raw text body. + """ + if isinstance(body, list) and body: + return _extract_provider_detail(body[0]) + if isinstance(body, dict): + detail = body.get("detail") + if isinstance(detail, str) and detail: + return detail + error = body.get("error") + if isinstance(error, dict): + message = error.get("message") + if isinstance(message, str) and message: + return message + message = body.get("message") + if isinstance(message, str) and message: + return message + if isinstance(body, str) and body.strip(): + return body.strip()[:2000] + return None + + +def raise_for_provider_http_error(error: UiPathAPIError) -> NoReturn: + """Convert a normalized ``UiPathAPIError`` into a structured ``AgentRuntimeError``. + + Reads the HTTP status code and the error message from ``error.body`` (the + gateway's ``detail`` envelope or the provider's own error envelope) and + re-raises as an ``AgentRuntimeError`` chained on the original. A 400 is the + caller's request/configuration, so it is categorised USER and surfaced + unwrapped; other unknown statuses keep the generic UNKNOWN wrapping. + """ + status_code = error.status_code + code = _LLM_STATUS_CODE_MAP.get(status_code, AgentRuntimeErrorCode.HTTP_ERROR) + if status_code == 403: + category = UiPathErrorCategory.DEPLOYMENT + elif status_code == 400: + category = UiPathErrorCategory.USER + else: + category = UiPathErrorCategory.UNKNOWN + detail = _extract_provider_detail(error.body) + + raise AgentRuntimeError( + code=code, + title=f"LLM provider returned HTTP {status_code}", + detail=detail or error.message or str(error), + category=category, + status=status_code, + ) from error diff --git a/src/uipath_langchain/agent/react/agent.py b/src/uipath_langchain/agent/react/agent.py index cee6c231f..20dda70b9 100644 --- a/src/uipath_langchain/agent/react/agent.py +++ b/src/uipath_langchain/agent/react/agent.py @@ -187,7 +187,6 @@ def create_agent( input_schema=input_schema, is_conversational=config.is_conversational, llm_messages_limit=config.llm_messages_limit, - thinking_messages_limit=config.thinking_messages_limit, tool_choice=config.tool_choice, parallel_tool_calls=config.parallel_tool_calls, strict_mode=config.strict_mode, @@ -219,7 +218,6 @@ def create_agent( ] route_agent = create_route_agent( valid_targets=target_node_names, - thinking_messages_limit=config.thinking_messages_limit, ) builder.add_conditional_edges( diff --git a/src/uipath_langchain/agent/react/constants.py b/src/uipath_langchain/agent/react/constants.py index d28d7b789..d6381503e 100644 --- a/src/uipath_langchain/agent/react/constants.py +++ b/src/uipath_langchain/agent/react/constants.py @@ -1,4 +1,3 @@ -DEFAULT_MAX_CONSECUTIVE_THINKING_MESSAGES = 0 DEFAULT_MAX_LLM_MESSAGES = 25 UIPATH_CONVERSATIONAL_AGENT_RESPONSE_MESSAGES_FIELD = "uipath__agent_response_messages" diff --git a/src/uipath_langchain/agent/react/conversational_output_node.py b/src/uipath_langchain/agent/react/conversational_output_node.py index 761412fc3..339052062 100644 --- a/src/uipath_langchain/agent/react/conversational_output_node.py +++ b/src/uipath_langchain/agent/react/conversational_output_node.py @@ -23,8 +23,10 @@ from uipath_langchain.chat.handlers import get_payload_handler from ..exceptions import AgentRuntimeError, AgentRuntimeErrorCode -from ..exceptions.licensing import raise_for_provider_http_error -from ..exceptions.llm import raise_for_llm_client_error +from ..exceptions.llm import ( + raise_for_llm_client_error, + raise_for_provider_http_error, +) from ..tools.utils import config_without_streaming from .tools.tools import create_set_conversational_output_tool from .types import AgentGraphState diff --git a/src/uipath_langchain/agent/react/forced_extraction.py b/src/uipath_langchain/agent/react/forced_extraction.py new file mode 100644 index 000000000..a2170ebfc --- /dev/null +++ b/src/uipath_langchain/agent/react/forced_extraction.py @@ -0,0 +1,73 @@ +"""Force a structured end_execution out of a thinking model that stalled. + +Anthropic won't honor a forced tool_choice while thinking is on, so a thinking model can +answer in plain text and never call end_execution. build_extraction_call retries that +turn with thinking off and the tool call forced, which every provider honors. +""" + +from typing import Any + +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import AIMessage, AnyMessage, HumanMessage +from uipath.agent.react import END_EXECUTION_TOOL + +from uipath_langchain.chat.thinking import is_reasoning_block, strip_thinking + +_END_EXECUTION_NAME = getattr( + END_EXECUTION_TOOL.name, "value", str(END_EXECUTION_TOOL.name) +) + + +def _strip_reasoning_blocks(messages: list[AnyMessage]) -> list[AnyMessage]: + """Drop reasoning blocks from AI messages (keep text + tool calls). + + They can't be replayed on a thinking-off call — an orphaned thinking block 400s. + """ + stripped: list[AnyMessage] = [] + for message in messages: + if isinstance(message, AIMessage) and isinstance(message.content, list): + kept = [ + block for block in message.content if not is_reasoning_block(block) + ] + if len(kept) != len(message.content): + # a turn that was only reasoning is now empty — drop it + if not kept and not message.tool_calls: + continue + message = message.model_copy(update={"content": kept}) + stripped.append(message) + return stripped + + +def _with_extraction_nudge(messages: list[AnyMessage]) -> list[AnyMessage]: + """Append (or merge into) a trailing user turn telling the model to call a tool. + + The wording is tool-neutral: continue the task, or finish with end_execution — so a + multi-tool agent that stalled mid-task isn't pushed to end early. Has to end on a user + turn: native/Vertex rejects a forced call that ends on the stalled assistant turn (a + prefill). Merge instead of appending so roles stay alternating even if the stalled turn + was dropped as empty. + """ + nudge = ( + f"Call the tool to continue the task. If you've finished, call " + f"{_END_EXECUTION_NAME} with the result." + ) + if messages and isinstance(messages[-1], HumanMessage): + last = messages[-1] + if isinstance(last.content, str): + merged: Any = f"{last.content}\n\n{nudge}" if last.content else nudge + elif isinstance(last.content, list): + merged = list(last.content) + [{"type": "text", "text": nudge}] + else: + merged = nudge + return list(messages[:-1]) + [HumanMessage(content=merged)] + return list(messages) + [HumanMessage(content=nudge)] + + +def build_extraction_call( + model: BaseChatModel, messages: list[AnyMessage] +) -> tuple[BaseChatModel, list[AnyMessage]]: + """The (model, messages) for the extraction call: thinking off, reasoning blocks + dropped, and a nudge to call end_execution — the caller then forces tool_choice.""" + return strip_thinking(model), _with_extraction_nudge( + _strip_reasoning_blocks(messages) + ) diff --git a/src/uipath_langchain/agent/react/llm_node.py b/src/uipath_langchain/agent/react/llm_node.py index 7a7a8fe24..7af7e202f 100644 --- a/src/uipath_langchain/agent/react/llm_node.py +++ b/src/uipath_langchain/agent/react/llm_node.py @@ -16,18 +16,19 @@ from uipath.runtime.errors import UiPathErrorCategory from uipath_langchain.chat.handlers import get_payload_handler +from uipath_langchain.chat.thinking import thinking_rejects_forced_tool_choice from ..exceptions import AgentRuntimeError, AgentRuntimeErrorCode -from ..exceptions.licensing import raise_for_provider_http_error -from ..exceptions.llm import raise_for_llm_client_error +from ..exceptions.llm import ( + raise_for_llm_client_error, + raise_for_provider_http_error, +) from ..messages.message_utils import replace_tool_calls from ..tools.static_args import StaticArgsHandler -from .constants import ( - DEFAULT_MAX_CONSECUTIVE_THINKING_MESSAGES, - DEFAULT_MAX_LLM_MESSAGES, -) +from .constants import DEFAULT_MAX_LLM_MESSAGES +from .forced_extraction import build_extraction_call from .types import FLOW_CONTROL_TOOLS, AgentGraphState -from .utils import count_consecutive_thinking_messages +from .utils import count_consecutive_tool_less_turns def _filter_control_flow_tool_calls( @@ -63,23 +64,24 @@ def create_llm_node( input_schema: type[InputT] | None = None, is_conversational: bool = False, llm_messages_limit: int = DEFAULT_MAX_LLM_MESSAGES, - thinking_messages_limit: int = DEFAULT_MAX_CONSECUTIVE_THINKING_MESSAGES, tool_choice: Literal["auto", "any"] = "auto", parallel_tool_calls: bool = True, strict_mode: bool = False, ): """Create LLM node with dynamic tool_choice enforcement. - Controls when to force tool usage based on consecutive thinking steps - to prevent infinite loops and ensure progress. + Forces tool usage every turn to keep the agent making progress. Forcing can + be silently downgraded on any transport (Bedrock handlers under thinking, + langchain_anthropic) or ignored by a BYOM deployment, so a tool-less turn is + tolerated: Anthropic thinking models retry it via the forced-extraction call + (thinking off, which every provider honors), and a second stall raises + THINKING_LIMIT_EXCEEDED. Args: model: The chat model to use tools: Available tools to bind is_conversational: Whether this is a conversational agent llm_messages_limit: Maximum number of LLM calls allowed per execution - thinking_messages_limit: Max consecutive LLM responses without tool calls - before enforcing tool usage. 0 = force tools every time. """ bindable_tools = list(tools) if tools else [] payload_handler = get_payload_handler(model) @@ -103,25 +105,39 @@ async def llm_node(state: StateT): static_schema_tools = static_args_handler.initialize( bindable_tools, state, input_schema or type(state) ) + current_tool_choice: Literal["auto", "any"] = tool_choice - if current_tool_choice == "auto" and ( - not is_conversational - and bindable_tools - and count_consecutive_thinking_messages(messages) >= thinking_messages_limit - ): + consecutive_tool_less = count_consecutive_tool_less_turns(messages) + thinking_rejects_forcing = thinking_rejects_forced_tool_choice(model) + call_model: BaseChatModel = model + call_messages: list[AnyMessage] = messages + handler = payload_handler + if not is_conversational and bindable_tools: + # only one tool_choice=auto call that doesnt return tool is allowed + if consecutive_tool_less > 1: + raise AgentRuntimeError( + code=AgentRuntimeErrorCode.THINKING_LIMIT_EXCEEDED, + title="Agent kept responding without calling a tool.", + detail="The model produced consecutive responses without tool calls " + "even after the forced extraction retry. If you are using a BYOM " + "configuration, verify your model deployment respects tool_choice.", + category=UiPathErrorCategory.SYSTEM, + ) current_tool_choice = "any" + if thinking_rejects_forcing and consecutive_tool_less > 0: + call_model, call_messages = build_extraction_call(model, messages) + handler = get_payload_handler(call_model) - binding_kwargs = payload_handler.get_tool_binding_kwargs( + binding_kwargs = handler.get_tool_binding_kwargs( tools=static_schema_tools, tool_choice=current_tool_choice, parallel_tool_calls=parallel_tool_calls, strict_mode=strict_mode, ) - - llm = model.bind_tools(static_schema_tools, **binding_kwargs) + llm = call_model.bind_tools(static_schema_tools, **binding_kwargs) try: - response = await llm.ainvoke(messages) + response = await llm.ainvoke(call_messages) except UiPathAPIError as e: # New LLM clients surface provider HTTP errors as a normalized UiPathAPIError directly. raise_for_provider_http_error(e) diff --git a/src/uipath_langchain/agent/react/router.py b/src/uipath_langchain/agent/react/router.py index 9d83ee743..eb30c1fd4 100644 --- a/src/uipath_langchain/agent/react/router.py +++ b/src/uipath_langchain/agent/react/router.py @@ -8,21 +8,19 @@ from ..exceptions import AgentRuntimeError, AgentRuntimeErrorCode from .types import FLOW_CONTROL_TOOLS, AgentGraphNode, AgentGraphState from .utils import ( - count_consecutive_thinking_messages, extract_current_tool_call_index, find_latest_ai_message, ) def create_route_agent( - thinking_messages_limit: int = 0, valid_targets: Container[str] | None = None, ): - """Create a routing function configured with thinking_messages_limit. + """Create the conditional-edge routing function. Args: - thinking_messages_limit: Max consecutive thinking messages before error valid_targets: Allowed routing destinations + Returns: Routing function for LangGraph conditional edges """ @@ -38,6 +36,11 @@ def route_agent( 3. If current tool call is a flow control tool, route to TERMINATE 4. Otherwise, route to the specific tool node + A tool-less turn with content always loops back to AGENT: the router can't + tell whether tool_choice was actually forced on the wire (handlers silently + downgrade it under thinking), so the LLM node owns stall accounting — + forcing, the extraction retry, and the deterministic failure. + Returns: - str: Single tool node name for sequential execution - AgentGraphNode.AGENT: When all tool calls completed or no tool calls @@ -57,28 +60,14 @@ def route_agent( ) if not last_message.tool_calls: - consecutive_thinking_messages = count_consecutive_thinking_messages( - messages - ) - - if consecutive_thinking_messages > thinking_messages_limit: - raise AgentRuntimeError( - code=AgentRuntimeErrorCode.THINKING_LIMIT_EXCEEDED, - title="Agent exceeded consecutive completions limit without producing tool calls.", - detail=f"Completions: {consecutive_thinking_messages}, max: {thinking_messages_limit}. " - f"This should not happen as tool_choice='required' is enforced at the limit." - "If you are using a BYOM configuration, verify your model deployment respects tool_choice or equivalent.", - category=UiPathErrorCategory.SYSTEM, - ) - if last_message.content: return AgentGraphNode.AGENT raise AgentRuntimeError( code=AgentRuntimeErrorCode.ROUTING_ERROR, title="Agent produced empty response without tool calls.", - detail=f"Consecutive completions: {consecutive_thinking_messages}, has_content: False." - "If you are using a BYOM configuration, verify your model deployment", + detail="The model returned no content and no tool calls. " + "If you are using a BYOM configuration, verify your model deployment.", category=UiPathErrorCategory.SYSTEM, ) diff --git a/src/uipath_langchain/agent/react/utils.py b/src/uipath_langchain/agent/react/utils.py index e2fa6d2c7..343ca5f80 100644 --- a/src/uipath_langchain/agent/react/utils.py +++ b/src/uipath_langchain/agent/react/utils.py @@ -105,8 +105,12 @@ def extract_input_data_from_state( return input_model.model_validate(filtered_state, from_attributes=True).model_dump() -def count_consecutive_thinking_messages(messages: Sequence[BaseMessage]) -> int: - """Count consecutive AIMessages without tool calls at end of message history.""" +def count_consecutive_tool_less_turns(messages: Sequence[BaseMessage]) -> int: + """Count trailing AI turns that produced content but called no tool. + + This is the stall counter: consecutive AIMessages at the end of the history with + content and no tool_calls, i.e. the model answered without making progress via a tool. + """ if not messages: return 0 diff --git a/src/uipath_langchain/chat/chat_model_factory.py b/src/uipath_langchain/chat/chat_model_factory.py index 112ebb4ca..bb438321f 100644 --- a/src/uipath_langchain/chat/chat_model_factory.py +++ b/src/uipath_langchain/chat/chat_model_factory.py @@ -10,6 +10,8 @@ before the ``uipath_langchain_client`` migration. """ +import logging +from collections.abc import Mapping from typing import Any, Final from langchain_core.callbacks import BaseCallbackHandler, Callbacks @@ -35,6 +37,8 @@ AgentStartupErrorCode, ) +logger = logging.getLogger(__name__) + class _TraceContextHeadersCallback(BaseCallbackHandler): """Inject W3C-style trace context headers into each LLM gateway request. @@ -87,6 +91,7 @@ def get_chat_model( callbacks: Callbacks = _UNSET, agenthub_config: str | None = None, use_new_llm_clients: bool = True, + model_settings: Mapping[str, Any] | None = None, **kwargs: Any, ) -> BaseChatModel: """Create and configure a chat model, dispatching legacy vs new clients. @@ -134,6 +139,13 @@ def get_chat_model( try: if not use_new_llm_clients: + if model_settings: + logger.warning( + "model_settings %s are not supported by the legacy LLM clients " + "and will be ignored; the EnabledNewLlmClients feature flag must " + "be on for them to apply.", + sorted(model_settings), + ) return _legacy_chat_model( model, temperature=temperature, @@ -164,6 +176,7 @@ def get_chat_model( api_flavor=api_flavor, custom_class=custom_class, agenthub_config=agenthub_config, + model_settings=model_settings, **optional_kwargs, **kwargs, ) diff --git a/src/uipath_langchain/chat/handlers/bedrock.py b/src/uipath_langchain/chat/handlers/bedrock.py index 7fe2ee180..a3124245e 100644 --- a/src/uipath_langchain/chat/handlers/bedrock.py +++ b/src/uipath_langchain/chat/handlers/bedrock.py @@ -9,6 +9,7 @@ from uipath.runtime.errors import UiPathErrorCategory from ..exceptions import ChatModelError, ChatModelErrorCode +from ..thinking import thinking_rejects_forced_tool_choice from .base import ModelPayloadHandler logger = logging.getLogger(__name__) @@ -86,15 +87,10 @@ def get_tool_binding_kwargs( parallel_tool_calls: bool | None = None, strict_mode: bool | None = None, ) -> dict[str, Any]: - _thinking = (getattr(self.model, "model_kwargs", None) or {}).get("thinking") - thinking_enabled = ( - isinstance(_thinking, dict) and _thinking.get("type") == "enabled" - ) - # Anthropic models via Invoke API don't support forced tool use with extended thinking - if thinking_enabled and tool_choice == "any": + if tool_choice == "any" and thinking_rejects_forced_tool_choice(self.model): logger.warning( - "Thinking is enabled for the model, but tool_choice is 'any'. " - "Changing tool_choice to 'auto' to keep the same behaviour as ChatAnthropicBedrock." + "Bedrock rejects forced tool_choice while thinking is active; " + "downgrading tool_choice 'any' -> 'auto'." ) tool_choice = "auto" kwargs: dict[str, Any] = {"tool_choice": tool_choice} @@ -142,17 +138,10 @@ def get_tool_binding_kwargs( parallel_tool_calls: bool | None = None, strict_mode: bool | None = None, ) -> dict[str, Any]: - _thinking = ( - getattr(self.model, "additional_model_request_fields", None) or {} - ).get("thinking") - thinking_enabled = ( - isinstance(_thinking, dict) and _thinking.get("type") == "enabled" - ) - # Anthropic models via Converse API don't support forced tool use with extended thinking - if thinking_enabled and tool_choice == "any": + if tool_choice == "any" and thinking_rejects_forced_tool_choice(self.model): logger.warning( - "Thinking is enabled for the model, but tool_choice is 'any'. " - "Changing tool_choice to 'auto' to keep the same behaviour as ChatAnthropicBedrock." + "Bedrock rejects forced tool_choice while thinking is active; " + "downgrading tool_choice 'any' -> 'auto'." ) tool_choice = "auto" kwargs: dict[str, Any] = {"tool_choice": tool_choice} diff --git a/src/uipath_langchain/chat/thinking.py b/src/uipath_langchain/chat/thinking.py new file mode 100644 index 000000000..afa2f62d7 --- /dev/null +++ b/src/uipath_langchain/chat/thinking.py @@ -0,0 +1,77 @@ +"""Anthropic thinking/reasoning knowledge shared across transports. + +Where the thinking config lives depends on the transport: native ChatAnthropic exposes a +`thinking` attribute, Bedrock Invoke nests it under `model_kwargs`, Bedrock Converse under +`additional_model_request_fields`. These helpers hide that so the payload handlers and the +ReAct loop don't each reimplement it. +""" + +from typing import Any + +from langchain_core.language_models import BaseChatModel + +_REASONING_BLOCK_TYPES = frozenset( + {"reasoning_content", "reasoning", "thinking", "redacted_thinking"} +) + + +def is_reasoning_block(block: Any) -> bool: + """True if a message content block is provider reasoning (thinking) output.""" + return isinstance(block, dict) and block.get("type") in _REASONING_BLOCK_TYPES + + +def thinking_rejects_forced_tool_choice(model: Any) -> bool: + """True if forcing a tool call is incompatible with this model's active thinking. + + Anthropic models (native + Bedrock) reject a forced tool_choice while extended or + adaptive thinking is on, so callers downgrade forcing to 'auto' and rely on the + thinking-off extraction retry (agent/react/forced_extraction.py). OpenAI/Gemini + reasoning tolerate forcing, so they return False. `{"type": "disabled"}` is thinking + off. Extend here if another provider shows the same conflict. + """ + for thinking in _thinking_configs(model): + if isinstance(thinking.get("type"), str): + return thinking["type"] != "disabled" + return False + + +def strip_thinking(model: BaseChatModel) -> BaseChatModel: + """Copy of the model with thinking config stripped, so forcing is honored. + + Adaptive thinking also carries an `output_config` effort knob on Converse; drop it too. + """ + updates: dict[str, object] = {} + request_fields = getattr(model, "additional_model_request_fields", None) + if isinstance(request_fields, dict) and ( + "thinking" in request_fields or "output_config" in request_fields + ): + updates["additional_model_request_fields"] = { + k: v + for k, v in request_fields.items() + if k not in ("thinking", "output_config") + } + model_kwargs = getattr(model, "model_kwargs", None) + if isinstance(model_kwargs, dict) and "thinking" in model_kwargs: + updates["model_kwargs"] = { + k: v for k, v in model_kwargs.items() if k != "thinking" + } + if getattr(model, "thinking", None) is not None: + updates["thinking"] = None + if not updates: + return model + try: + return model.model_copy(update=updates) + except Exception: + return model + + +def _thinking_configs(model: Any) -> list[dict[str, Any]]: + """The thinking dicts set on a model, read from every transport's location.""" + invoke = getattr(model, "model_kwargs", None) or {} + converse = getattr(model, "additional_model_request_fields", None) or {} + candidates = ( + getattr(model, "thinking", None), + invoke.get("thinking") if isinstance(invoke, dict) else None, + converse.get("thinking") if isinstance(converse, dict) else None, + ) + return [c for c in candidates if isinstance(c, dict)] diff --git a/tests/agent/react/test_forced_extraction.py b/tests/agent/react/test_forced_extraction.py new file mode 100644 index 000000000..5e333eaa1 --- /dev/null +++ b/tests/agent/react/test_forced_extraction.py @@ -0,0 +1,157 @@ +"""Tests for the forced-extraction helpers.""" + +from typing import Any + +from langchain_core.messages import AIMessage, HumanMessage +from langchain_core.messages.content import create_tool_call +from pydantic import BaseModel, ConfigDict + +from uipath_langchain.agent.react.forced_extraction import ( + _strip_reasoning_blocks, + _with_extraction_nudge, +) +from uipath_langchain.chat.thinking import strip_thinking + + +class _FakeConverse(BaseModel): + """Stand-in for ChatBedrockConverse: thinking lives in additional_model_request_fields.""" + + model_config = ConfigDict(extra="allow") + additional_model_request_fields: dict[str, Any] | None = None + + +class _FakeInvoke(BaseModel): + """Stand-in for ChatBedrock (Invoke): thinking lives in model_kwargs.""" + + model_config = ConfigDict(extra="allow") + model_kwargs: dict[str, Any] = {} + + +class _FakeNative(BaseModel): + """Stand-in for ChatAnthropic: thinking is a top-level attribute.""" + + model_config = ConfigDict(extra="allow") + thinking: dict[str, Any] | None = None + + +class TestStripThinking: + """strip_thinking removes reasoning config across transports, keeping the rest.""" + + def test_converse_removes_thinking_keeps_others(self) -> None: + model = _FakeConverse( + additional_model_request_fields={ + "thinking": {"type": "enabled", "budget_tokens": 2048}, + "anthropic_beta": ["x"], + } + ) + result = strip_thinking(model) # type: ignore[arg-type] + assert result.additional_model_request_fields == {"anthropic_beta": ["x"]} + + def test_converse_removes_thinking_and_output_config(self) -> None: + model = _FakeConverse( + additional_model_request_fields={ + "thinking": {"type": "adaptive"}, + "output_config": {"effort": "high"}, + } + ) + result = strip_thinking(model) # type: ignore[arg-type] + assert result.additional_model_request_fields == {} + + def test_invoke_removes_thinking_from_model_kwargs(self) -> None: + model = _FakeInvoke( + model_kwargs={"thinking": {"type": "enabled"}, "top_p": 0.9} + ) + result = strip_thinking(model) # type: ignore[arg-type] + assert result.model_kwargs == {"top_p": 0.9} + + def test_native_clears_thinking_attribute(self) -> None: + model = _FakeNative(thinking={"type": "adaptive"}) + result = strip_thinking(model) # type: ignore[arg-type] + assert result.thinking is None + + def test_no_thinking_returns_same_instance(self) -> None: + model = _FakeConverse(additional_model_request_fields={"anthropic_beta": ["x"]}) + assert strip_thinking(model) is model # type: ignore[arg-type] + + +class TestStripReasoningBlocks: + """_strip_reasoning_blocks drops reasoning blocks, keeps text and tool calls.""" + + def test_keeps_text_drops_reasoning(self) -> None: + msg = AIMessage( + content=[ + {"type": "reasoning_content", "reasoning_content": {"text": "think"}}, + {"type": "text", "text": "answer"}, + ] + ) + out = _strip_reasoning_blocks([msg]) + assert len(out) == 1 + assert out[0].content == [{"type": "text", "text": "answer"}] + + def test_drops_reasoning_only_turn(self) -> None: + human = HumanMessage(content="q") + reasoning_only = AIMessage( + content=[{"type": "reasoning_content", "reasoning_content": {"text": "t"}}] + ) + out = _strip_reasoning_blocks([human, reasoning_only]) + assert out == [human] + + def test_keeps_turn_with_tool_call_even_if_content_empties(self) -> None: + msg = AIMessage( + content=[{"type": "reasoning_content", "reasoning_content": {"text": "t"}}], + tool_calls=[create_tool_call(name="end_execution", args={}, id="call_1")], + ) + out = _strip_reasoning_blocks([msg]) + assert len(out) == 1 + assert out[0].content == [] + assert out[0].tool_calls[0]["name"] == "end_execution" + + def test_strips_redacted_thinking_blocks(self) -> None: + """redacted_thinking can't be replayed on a thinking-off call either.""" + msg = AIMessage( + content=[ + {"type": "redacted_thinking", "data": "opaque"}, + {"type": "text", "text": "answer"}, + ] + ) + out = _strip_reasoning_blocks([msg]) + assert out[0].content == [{"type": "text", "text": "answer"}] + + def test_string_content_untouched(self) -> None: + msg = AIMessage(content="plain answer") + out = _strip_reasoning_blocks([msg]) + assert out[0] is msg + + def test_non_ai_messages_untouched(self) -> None: + human = HumanMessage(content="q") + out = _strip_reasoning_blocks([human]) + assert out == [human] + + +class TestExtractionNudge: + """_with_extraction_nudge ends on a user turn without creating consecutive user turns.""" + + def test_appends_user_turn_after_assistant(self) -> None: + msgs = [HumanMessage(content="q"), AIMessage(content="answer")] + out = _with_extraction_nudge(msgs) + assert isinstance(out[-1], HumanMessage) + # tool-neutral: mentions continuing, not only end_execution, so a mid-task + # stall isn't pushed to terminate early + assert "continue" in out[-1].content + assert "end_execution" in out[-1].content + assert isinstance(out[-2], AIMessage) + + def test_merges_into_trailing_user_turn(self) -> None: + msgs = [HumanMessage(content="the task")] + out = _with_extraction_nudge(msgs) + assert len(out) == 1 + assert isinstance(out[-1], HumanMessage) + assert "the task" in out[-1].content + assert "end_execution" in out[-1].content + + def test_merges_into_list_content_user_turn(self) -> None: + msgs = [HumanMessage(content=[{"type": "text", "text": "the task"}])] + out = _with_extraction_nudge(msgs) + assert len(out) == 1 + assert out[-1].content[-1]["type"] == "text" + assert "end_execution" in out[-1].content[-1]["text"] diff --git a/tests/agent/react/test_llm_node.py b/tests/agent/react/test_llm_node.py index 12eb5967d..49ecb8c8a 100644 --- a/tests/agent/react/test_llm_node.py +++ b/tests/agent/react/test_llm_node.py @@ -1,7 +1,7 @@ """Tests for LLM node tool call filtering functionality.""" from typing import Any -from unittest.mock import AsyncMock, Mock +from unittest.mock import AsyncMock, Mock, patch import httpx import openai @@ -441,3 +441,151 @@ async def test_non_http_error_propagates_unchanged(self): with pytest.raises(ValueError, match="boom"): await node(self.state) + + +class TestForcedExtractionEscalation: + """Stall accounting in the LLM node. Tool usage is forced every turn. Anthropic + thinking models can't be plain-forced (turn 1's forced `any` is downgraded to `auto` + by the handler so they reason), so their first stall retries via a thinking-off + extraction call; every other provider forces from turn 1 and is never extracted + (forcing works for it, and the extraction's reasoning-strip would break it, e.g. + OpenAI 400s on a continuation with its reasoning removed). A second stall fails + deterministically. Not gated on tool_choice='auto', since forcing can be downgraded + on the wire (Bedrock handlers, langchain_anthropic).""" + + def _thinking_model(self) -> Any: + model = _StubAzureChatOpenAI.model_construct() + model.thinking = { + "type": "adaptive" + } # thinking active; OpenAI MRO won't downgrade + model.bind_tools = Mock(return_value=model) + model.bind = Mock(return_value=model) + return model + + def _plain_model(self) -> Any: + model = _StubAzureChatOpenAI.model_construct() + model.bind_tools = Mock(return_value=model) + model.bind = Mock(return_value=model) + return model + + def _stalled_state(self, stalls: int = 1) -> AgentGraphState: + prior = AIMessage( + content=[ + {"type": "reasoning_content", "reasoning_content": {"text": "think"}}, + {"type": "text", "text": "answer"}, + ] + ) + return AgentGraphState( + messages=[HumanMessage(content="q"), *([prior] * stalls)] + ) + + async def _run_capture(self, model: Any, tool_choice: str = "auto") -> list[Any]: + captured: dict[str, Any] = {} + + async def fake_ainvoke(msgs: Any) -> AIMessage: + captured["msgs"] = msgs + return AIMessage( + content="", + tool_calls=[ + create_tool_call(name=END_EXECUTION_TOOL.name, args={}, id="c1") + ], + ) + + model.ainvoke = AsyncMock(side_effect=fake_ainvoke) + tool = Mock(spec=BaseTool) + tool.name = "t" + node = create_llm_node(model, [tool], tool_choice=tool_choice) + await node(self._stalled_state()) + return captured["msgs"] + + @pytest.mark.asyncio + async def test_thinking_model_stall_strips_reasoning_and_nudges(self) -> None: + """Escalation fires: reasoning stripped from the stalled turn, and the request + ends on a user nudge (not an assistant prefill) so native accepts the forced call.""" + msgs = await self._run_capture(self._thinking_model()) + # stalled assistant turn: reasoning block gone, text kept + assert msgs[-2].content == [{"type": "text", "text": "answer"}] + # request ends on a user instruction to emit the terminal tool call + assert isinstance(msgs[-1], HumanMessage) + assert "end_execution" in msgs[-1].content + + @pytest.mark.asyncio + async def test_plain_model_stall_is_forced_not_extracted(self) -> None: + """A non-Anthropic model that stalls is plain-forced, never extracted: forcing + works for it directly, and the extraction's reasoning-strip would break it.""" + model = self._plain_model() + model.ainvoke = AsyncMock( + return_value=AIMessage( + content="", + tool_calls=[ + create_tool_call(name=END_EXECUTION_TOOL.name, args={}, id="c1") + ], + ) + ) + tool = Mock(spec=BaseTool) + tool.name = "t" + with patch( + "uipath_langchain.agent.react.llm_node.build_extraction_call" + ) as spy: + await create_llm_node(model, [tool])(self._stalled_state()) + spy.assert_not_called() + assert model.bind_tools.call_args.kwargs["tool_choice"] == "any" + + @pytest.mark.asyncio + async def test_escalates_when_tool_choice_configured_any(self) -> None: + """AgentGraphConfig(tool_choice='any') keeps the termination guarantee: the + extraction retry must not be gated on tool_choice starting as 'auto'.""" + msgs = await self._run_capture(self._thinking_model(), tool_choice="any") + assert isinstance(msgs[-1], HumanMessage) + assert "end_execution" in msgs[-1].content + + @pytest.mark.asyncio + async def test_no_stall_does_not_escalate(self) -> None: + """First turn (no prior tool-less turn) must not force extraction.""" + model = self._thinking_model() + model.ainvoke = AsyncMock(return_value=AIMessage(content="reasoning...")) + state = AgentGraphState(messages=[HumanMessage(content="q")]) + tool = Mock(spec=BaseTool) + tool.name = "t" + with patch( + "uipath_langchain.agent.react.llm_node.build_extraction_call" + ) as spy: + await create_llm_node(model, [tool])(state) + spy.assert_not_called() + + @pytest.mark.asyncio + async def test_second_stall_after_extraction_raises_thinking_limit(self) -> None: + """A model that stalls again after the forced-extraction retry fails fast + with a SYSTEM diagnostic instead of looping to the max-iterations limit.""" + model = self._thinking_model() + model.ainvoke = AsyncMock(return_value=AIMessage(content="still stalling")) + tool = Mock(spec=BaseTool) + tool.name = "t" + node = create_llm_node(model, [tool]) + + with pytest.raises(AgentRuntimeError) as exc_info: + await node(self._stalled_state(stalls=2)) + + info = exc_info.value.error_info + assert info.code.endswith(AgentRuntimeErrorCode.THINKING_LIMIT_EXCEEDED.value) + assert info.category == UiPathErrorCategory.SYSTEM + model.ainvoke.assert_not_awaited() + + @pytest.mark.asyncio + async def test_plain_model_forces_from_first_turn(self) -> None: + """A non-Anthropic model forces tool_choice from turn 1 (it reasons and calls + the tool in the same forced turn), so no reasoning buffer is needed.""" + model = self._plain_model() + model.ainvoke = AsyncMock( + return_value=AIMessage( + content="", + tool_calls=[ + create_tool_call(name=END_EXECUTION_TOOL.name, args={}, id="c1") + ], + ) + ) + tool = Mock(spec=BaseTool) + tool.name = "t" + node = create_llm_node(model, [tool]) + await node(AgentGraphState(messages=[HumanMessage(content="q")])) + assert model.bind_tools.call_args.kwargs["tool_choice"] == "any" diff --git a/tests/agent/react/test_router.py b/tests/agent/react/test_router.py index 6627cd42d..79901ea4c 100644 --- a/tests/agent/react/test_router.py +++ b/tests/agent/react/test_router.py @@ -45,14 +45,14 @@ class MockAgentGraphState(BaseModel): @pytest.fixture def route_function_no_limit(): - """Fixture for routing function with no thinking messages limit.""" - return create_route_agent(valid_targets=_VALID_TARGETS, thinking_messages_limit=0) + """Routing function. Tool-less turns loop back; llm_messages_limit bounds them.""" + return create_route_agent(valid_targets=_VALID_TARGETS) @pytest.fixture def route_function_with_limit(): - """Fixture for routing function with thinking messages limit of 2.""" - return create_route_agent(valid_targets=_VALID_TARGETS, thinking_messages_limit=2) + """Alias kept for existing tests; routing no longer takes a thinking limit.""" + return create_route_agent(valid_targets=_VALID_TARGETS) @pytest.fixture @@ -141,18 +141,6 @@ def state_no_tool_calls(): return MockAgentGraphState(messages=[HumanMessage(content="query"), ai_message]) -@pytest.fixture -def state_excessive_thinking(): - """Fixture for state with excessive consecutive thinking messages.""" - messages = [ - HumanMessage(content="query"), - AIMessage(content="thinking 1"), - AIMessage(content="thinking 2"), - AIMessage(content="thinking 3"), - ] - return MockAgentGraphState(messages=messages) - - @pytest.fixture def empty_state(): """Fixture for state with no messages.""" @@ -205,62 +193,46 @@ def test_flow_control_tool_terminates( assert result == AgentGraphNode.TERMINATE -class TestRouteAgentThinkingMessages: - """Test thinking messages and consecutive completions logic.""" - - def test_no_tool_calls_within_limit_routes_to_agent( - self, route_function_with_limit, state_no_tool_calls - ): - """Should route to AGENT when no tool calls and within thinking limit.""" - result = route_function_with_limit(state_no_tool_calls) - assert result == AgentGraphNode.AGENT - - def test_excessive_thinking_messages_raises_exception( - self, route_function_with_limit, state_excessive_thinking - ): - """Should raise exception when exceeding thinking messages limit.""" - with pytest.raises(AgentRuntimeError) as exc_info: - route_function_with_limit(state_excessive_thinking) - - assert exc_info.value.error_info.code == AgentRuntimeError.full_code( - AgentRuntimeErrorCode.THINKING_LIMIT_EXCEEDED - ) +class TestRouteAgentToolLessTurns: + """Any tool-less content turn loops back to AGENT. The router can't tell whether + tool_choice was actually forced on the wire (Bedrock/native handlers silently + downgrade it under thinking), so stall accounting — forcing, the extraction + retry, and the deterministic failure — lives in the LLM node.""" - def test_thinking_messages_limit_zero_forbids_thinking(self): - """Should not allow any thinking messages when limit is 0.""" - route_func = create_route_agent( - valid_targets=_VALID_TARGETS, thinking_messages_limit=0 + def test_reasoning_stall_routes_to_agent(self, route_function_no_limit): + """A tool-less turn carrying a thinking block loops back to AGENT.""" + ai_message = AIMessage( + content=[ + {"type": "thinking", "thinking": "", "signature": "sig"}, + {"type": "text", "text": "the answer is 42"}, + ] ) - ai_message = AIMessage(content="thinking") state = MockAgentGraphState( messages=[HumanMessage(content="query"), ai_message] ) + assert route_function_no_limit(state) == AgentGraphNode.AGENT - with pytest.raises(AgentRuntimeError) as exc_info: - route_func(state) - - assert exc_info.value.error_info.code == AgentRuntimeError.full_code( - AgentRuntimeErrorCode.THINKING_LIMIT_EXCEEDED + def test_plain_content_without_reasoning_routes_to_agent( + self, route_function_no_limit, state_no_tool_calls + ): + """A plain-text tool-less turn is a legal reply whenever forcing was + downgraded (Bedrock thinking, adaptive thinking after tool results), so it + loops back instead of failing the run.""" + assert route_function_no_limit(state_no_tool_calls) == AgentGraphNode.AGENT + + def test_redacted_thinking_turn_routes_to_agent(self, route_function_no_limit): + """Redacted thinking turns carry no plain reasoning block but are still + legitimate reasoning stalls.""" + ai_message = AIMessage( + content=[ + {"type": "redacted_thinking", "data": "opaque"}, + {"type": "text", "text": "working on it"}, + ] ) - - def test_thinking_messages_after_tool_execution_resets_count(self): - """Should reset thinking count after tool execution.""" - route_func = create_route_agent( - valid_targets=_VALID_TARGETS, thinking_messages_limit=1 + state = MockAgentGraphState( + messages=[HumanMessage(content="query"), ai_message] ) - messages = [ - HumanMessage(content="query"), - AIMessage( - content="using tool", - tool_calls=[{"name": "test_tool", "args": {}, "id": "call_1"}], - ), - ToolMessage(content="result", tool_call_id="call_1"), - AIMessage(content="thinking after tool"), # This should be allowed - ] - state = MockAgentGraphState(messages=messages) - - result = route_func(state) - assert result == AgentGraphNode.AGENT + assert route_function_no_limit(state) == AgentGraphNode.AGENT class TestRouteAgentErrorHandling: @@ -310,7 +282,6 @@ def test_unknown_target_raises_routing_error(self): """Should raise ROUTING_ERROR (SYSTEM) when the routed tool is unwired.""" route_func = create_route_agent( valid_targets=[AgentGraphNode.AGENT, AgentGraphNode.TERMINATE, "real_tool"], - thinking_messages_limit=0, ) ai_message = AIMessage( content="routing", @@ -330,7 +301,6 @@ def test_known_target_returns_tool_name(self): """Should return the tool name when it is in the valid target set.""" route_func = create_route_agent( valid_targets=[AgentGraphNode.AGENT, AgentGraphNode.TERMINATE, "real_tool"], - thinking_messages_limit=0, ) ai_message = AIMessage( content="routing", @@ -346,7 +316,7 @@ def test_default_valid_targets_skips_guard(self): Backwards-compatible contract: callers predating valid_targets must keep the old unguarded behavior, returning any routed tool name as-is. """ - route_func = create_route_agent(thinking_messages_limit=0) + route_func = create_route_agent() ai_message = AIMessage( content="routing", tool_calls=[{"name": "unwired_tool", "args": {}, "id": "call_1"}], diff --git a/tests/agent/react/test_utils.py b/tests/agent/react/test_utils.py index c3142ab59..294db419e 100644 --- a/tests/agent/react/test_utils.py +++ b/tests/agent/react/test_utils.py @@ -12,7 +12,7 @@ ) from uipath_langchain.agent.react.utils import ( build_conversational_output_args_schema, - count_consecutive_thinking_messages, + count_consecutive_tool_less_turns, extract_current_tool_call_index, find_latest_ai_message, has_custom_conversational_output_fields, @@ -24,12 +24,12 @@ class TestCountSuccessiveCompletions: def test_empty_messages(self): """Should return 0 for empty message list.""" - assert count_consecutive_thinking_messages([]) == 0 + assert count_consecutive_tool_less_turns([]) == 0 def test_no_ai_messages(self): """Should return 0 when no AI messages exist.""" messages = [HumanMessage(content="test")] - assert count_consecutive_thinking_messages(messages) == 0 + assert count_consecutive_tool_less_turns(messages) == 0 def test_last_message_not_ai(self): """Should return 0 when last message is not AI.""" @@ -37,7 +37,7 @@ def test_last_message_not_ai(self): AIMessage(content="response"), HumanMessage(content="follow-up"), ] - assert count_consecutive_thinking_messages(messages) == 0 + assert count_consecutive_tool_less_turns(messages) == 0 def test_ai_message_with_tool_calls(self): """Should return 0 when last AI message has tool calls.""" @@ -48,7 +48,7 @@ def test_ai_message_with_tool_calls(self): tool_calls=[{"name": "test", "args": {}, "id": "call_1"}], ), ] - assert count_consecutive_thinking_messages(messages) == 0 + assert count_consecutive_tool_less_turns(messages) == 0 def test_ai_message_without_content(self): """Should return 0 when last AI message has no content.""" @@ -56,7 +56,7 @@ def test_ai_message_without_content(self): HumanMessage(content="query"), AIMessage(content=""), ] - assert count_consecutive_thinking_messages(messages) == 0 + assert count_consecutive_tool_less_turns(messages) == 0 def test_single_text_completion(self): """Should count single text-only AI message.""" @@ -64,7 +64,7 @@ def test_single_text_completion(self): HumanMessage(content="query"), AIMessage(content="thinking"), ] - assert count_consecutive_thinking_messages(messages) == 1 + assert count_consecutive_tool_less_turns(messages) == 1 def test_two_successive_completions(self): """Should count multiple consecutive text-only AI messages.""" @@ -73,7 +73,7 @@ def test_two_successive_completions(self): AIMessage(content="thinking 1"), AIMessage(content="thinking 2"), ] - assert count_consecutive_thinking_messages(messages) == 2 + assert count_consecutive_tool_less_turns(messages) == 2 def test_three_successive_completions(self): """Should count all consecutive text-only AI messages at end.""" @@ -83,7 +83,7 @@ def test_three_successive_completions(self): AIMessage(content="thinking 2"), AIMessage(content="thinking 3"), ] - assert count_consecutive_thinking_messages(messages) == 3 + assert count_consecutive_tool_less_turns(messages) == 3 def test_tool_call_resets_count(self): """Should only count completions after last tool call.""" @@ -98,7 +98,7 @@ def test_tool_call_resets_count(self): AIMessage(content="thinking 2"), AIMessage(content="thinking 3"), ] - assert count_consecutive_thinking_messages(messages) == 2 + assert count_consecutive_tool_less_turns(messages) == 2 def test_mixed_message_types(self): """Should handle complex message patterns correctly.""" @@ -114,7 +114,7 @@ def test_mixed_message_types(self): HumanMessage(content="user follow-up"), AIMessage(content="responding to follow-up"), ] - assert count_consecutive_thinking_messages(messages) == 1 + assert count_consecutive_tool_less_turns(messages) == 1 def test_multiple_tool_calls_in_message(self): """Should reset count even with multiple tool calls.""" @@ -129,7 +129,7 @@ def test_multiple_tool_calls_in_message(self): ], ), ] - assert count_consecutive_thinking_messages(messages) == 0 + assert count_consecutive_tool_less_turns(messages) == 0 def test_ai_message_with_empty_tool_calls_list(self): """Should handle AI message with empty tool_calls list.""" @@ -137,7 +137,7 @@ def test_ai_message_with_empty_tool_calls_list(self): HumanMessage(content="query"), AIMessage(content="thinking", tool_calls=[]), ] - assert count_consecutive_thinking_messages(messages) == 1 + assert count_consecutive_tool_less_turns(messages) == 1 def test_only_ai_messages_all_text(self): """Should count all AI messages when all are text-only.""" @@ -146,7 +146,7 @@ def test_only_ai_messages_all_text(self): AIMessage(content="thought 2"), AIMessage(content="thought 3"), ] - assert count_consecutive_thinking_messages(messages) == 3 + assert count_consecutive_tool_less_turns(messages) == 3 class TestFindLatestAiMessage: diff --git a/tests/agent/test_licensing.py b/tests/agent/test_llm.py similarity index 97% rename from tests/agent/test_licensing.py rename to tests/agent/test_llm.py index 8eb8870fa..5bee3268b 100644 --- a/tests/agent/test_licensing.py +++ b/tests/agent/test_llm.py @@ -14,7 +14,7 @@ AgentRuntimeError, AgentRuntimeErrorCode, ) -from uipath_langchain.agent.exceptions.licensing import raise_for_provider_http_error +from uipath_langchain.agent.exceptions.llm import raise_for_provider_http_error _DETAIL = "License not available for LLM usage. You need additional 'AGU'." diff --git a/tests/chat/handlers/test_tool_binding_kwargs.py b/tests/chat/handlers/test_tool_binding_kwargs.py index 59fee0115..ce0af6bd0 100644 --- a/tests/chat/handlers/test_tool_binding_kwargs.py +++ b/tests/chat/handlers/test_tool_binding_kwargs.py @@ -178,6 +178,24 @@ def test_all_keys_present(self): assert set(result.keys()) == {"tool_choice", "parallel_tool_calls", "strict"} +class TestAnthropicKeepsForcingUnderThinking: + """Native Anthropic accepts forced tool_choice under thinking (unlike Bedrock), + so it must NOT downgrade 'any' regardless of the thinking mode.""" + + def _model(self, thinking: object) -> object: + return type("FakeChatAnthropic", (), {"thinking": thinking})() + + def test_extended_thinking_keeps_any(self): + handler = AnthropicPayloadHandler(self._model({"type": "enabled"})) # type: ignore[arg-type] + result = handler.get_tool_binding_kwargs(tools=[], tool_choice="any") + assert result["tool_choice"] == "any" + + def test_adaptive_thinking_keeps_any(self): + handler = AnthropicPayloadHandler(self._model({"type": "adaptive"})) # type: ignore[arg-type] + result = handler.get_tool_binding_kwargs(tools=[], tool_choice="any") + assert result["tool_choice"] == "any" + + # --------------------------------------------------------------------------- # Gemini handler # --------------------------------------------------------------------------- diff --git a/tests/chat/test_bedrock_payload_handler.py b/tests/chat/test_bedrock_payload_handler.py index 02fdca468..6d018d395 100644 --- a/tests/chat/test_bedrock_payload_handler.py +++ b/tests/chat/test_bedrock_payload_handler.py @@ -10,25 +10,34 @@ BedrockConversePayloadHandler, BedrockInvokePayloadHandler, ) +from uipath_langchain.chat.thinking import thinking_rejects_forced_tool_choice # --------------------------------------------------------------------------- # Fake model factories # --------------------------------------------------------------------------- -def make_invoke_model(**model_kwargs_override: object) -> object: - """Return a ChatBedrock-like model with optional model_kwargs.""" +def make_invoke_model( + model_id: str | None = None, **model_kwargs_override: object +) -> object: + """Return a ChatBedrock-like model with optional model_kwargs + model_id.""" model = type("FakeChatBedrock", (), {"model_kwargs": {}})() model.model_kwargs = model_kwargs_override + if model_id is not None: + model.model_id = model_id return model -def make_converse_model(**fields_override: object) -> object: - """Return a ChatBedrockConverse-like model with optional additional_model_request_fields.""" +def make_converse_model( + model_id: str | None = None, **fields_override: object +) -> object: + """Return a ChatBedrockConverse-like model with optional request fields + model_id.""" model = type( "FakeChatBedrockConverse", (), {"additional_model_request_fields": {}} )() model.additional_model_request_fields = fields_override + if model_id is not None: + model.model_id = model_id return model @@ -298,3 +307,77 @@ def test_additional_fields_attribute_missing(self) -> None: handler = BedrockConversePayloadHandler(model) result = handler.get_tool_binding_kwargs([], "any") assert result["tool_choice"] == "any" + + +# --------------------------------------------------------------------------- +# Bedrock downgrades forced tool_choice whenever thinking is active — any mode, +# any version (no version check). The ReAct loop's forced-extraction fallback then +# guarantees termination. Native keeps forcing; see test_tool_binding_kwargs.py. +# --------------------------------------------------------------------------- + + +class TestBedrockThinkingDowngrade: + """Any thinking mode downgrades forced 'any' -> 'auto' on both Bedrock APIs.""" + + @pytest.mark.parametrize("mode", ["enabled", "adaptive"]) + def test_converse_downgrades(self, mode: str) -> None: + handler = BedrockConversePayloadHandler( + make_converse_model(thinking={"type": mode}) # type: ignore[arg-type] + ) + assert handler.get_tool_binding_kwargs([], "any")["tool_choice"] == "auto" + + @pytest.mark.parametrize("mode", ["enabled", "adaptive"]) + def test_invoke_downgrades(self, mode: str) -> None: + handler = BedrockInvokePayloadHandler( + make_invoke_model(thinking={"type": mode}) # type: ignore[arg-type] + ) + assert handler.get_tool_binding_kwargs([], "any")["tool_choice"] == "auto" + + def test_no_thinking_keeps_forcing(self) -> None: + handler = BedrockConversePayloadHandler(make_converse_model()) # type: ignore[arg-type] + assert handler.get_tool_binding_kwargs([], "any")["tool_choice"] == "any" + + def test_disabled_thinking_keeps_forcing_converse(self) -> None: + handler = BedrockConversePayloadHandler( + make_converse_model(thinking={"type": "disabled"}) # type: ignore[arg-type] + ) + assert handler.get_tool_binding_kwargs([], "any")["tool_choice"] == "any" + + def test_disabled_thinking_keeps_forcing_invoke(self) -> None: + handler = BedrockInvokePayloadHandler( + make_invoke_model(thinking={"type": "disabled"}) # type: ignore[arg-type] + ) + assert handler.get_tool_binding_kwargs([], "any")["tool_choice"] == "any" + + +class TestThinkingRejectsForcedToolChoice: + """The downgrade predicate: forcing is rejected iff active thinking is present, + read from whichever transport carries the thinking dict.""" + + @pytest.mark.parametrize( + "thinking_type,expected", + [ + ("enabled", True), + ("adaptive", True), + ("interleaved", True), + ("disabled", False), + ], + ) + def test_mode_decides_rejection(self, thinking_type: str, expected: bool) -> None: + model = make_invoke_model(thinking={"type": thinking_type}) + assert thinking_rejects_forced_tool_choice(model) is expected + + def test_reads_native_thinking_attribute(self) -> None: + model = type("FakeChatAnthropic", (), {"thinking": {"type": "adaptive"}})() + assert thinking_rejects_forced_tool_choice(model) is True + + def test_reads_converse_request_fields(self) -> None: + model = make_converse_model(thinking={"type": "enabled"}) + assert thinking_rejects_forced_tool_choice(model) is True + + def test_false_when_thinking_absent(self) -> None: + assert thinking_rejects_forced_tool_choice(make_invoke_model()) is False + + def test_false_when_thinking_not_dict(self) -> None: + model = make_invoke_model(thinking="enabled") + assert thinking_rejects_forced_tool_choice(model) is False diff --git a/tests/chat/test_chat_model_factory_model_settings.py b/tests/chat/test_chat_model_factory_model_settings.py new file mode 100644 index 000000000..18920731f --- /dev/null +++ b/tests/chat/test_chat_model_factory_model_settings.py @@ -0,0 +1,59 @@ +"""Tests for how ``chat_model_factory.get_chat_model`` dispatches ``model_settings``.""" + +import logging +from unittest.mock import MagicMock + +from uipath_langchain.chat.chat_model_factory import get_chat_model + + +class TestModelSettingsDispatch: + def test_new_path_forwards_model_settings(self, mocker): + upstream = mocker.patch( + "uipath_langchain.chat.chat_model_factory.get_chat_model_factory", + return_value=MagicMock(), + ) + + get_chat_model( + "gpt-4o", + use_new_llm_clients=True, + model_settings={"reasoning_effort": "high"}, + ) + + assert upstream.call_args.kwargs["model_settings"] == { + "reasoning_effort": "high" + } + + def test_legacy_path_warns_when_model_settings_dropped(self, mocker, caplog): + """The legacy clients can't apply model_settings; dropping them must be + loud so a tenant with EnableModelSpecificSettings on but + EnabledNewLlmClients off can be diagnosed from logs.""" + legacy = mocker.patch( + "uipath_langchain.chat.chat_model_factory._legacy_chat_model", + return_value=MagicMock(), + ) + + with caplog.at_level(logging.WARNING): + get_chat_model( + "gpt-4o", + agenthub_config="cfg", + use_new_llm_clients=False, + model_settings={"reasoning_effort": "high"}, + ) + + legacy.assert_called_once() + assert any("model_settings" in record.message for record in caplog.records) + + def test_legacy_path_silent_without_model_settings(self, mocker, caplog): + mocker.patch( + "uipath_langchain.chat.chat_model_factory._legacy_chat_model", + return_value=MagicMock(), + ) + + with caplog.at_level(logging.WARNING): + get_chat_model( + "gpt-4o", + agenthub_config="cfg", + use_new_llm_clients=False, + ) + + assert not any("model_settings" in record.message for record in caplog.records)