diff --git a/langfuse/langchain/CallbackHandler.py b/langfuse/langchain/CallbackHandler.py index dac76f4fc..01612cbad 100644 --- a/langfuse/langchain/CallbackHandler.py +++ b/langfuse/langchain/CallbackHandler.py @@ -1820,6 +1820,18 @@ def _parse_usage(response: LLMResult) -> Any: def _parse_model(response: LLMResult) -> Any: + # llm_output["model_name"] can be the configured alias, so prefer the + # model the provider reported on the generation + if response.generations and response.generations[-1]: + generation = response.generations[-1][-1] + message = getattr(generation, "message", None) + for metadata in ( + getattr(message, "response_metadata", None), + generation.generation_info, + ): + if isinstance(metadata, dict) and metadata.get("model_name"): + return metadata["model_name"] + # langchain-anthropic uses the usage field llm_model_keys = ["model_name"] llm_model = None diff --git a/tests/unit/test_langchain.py b/tests/unit/test_langchain.py index c55eaa3d2..f2a9e04a8 100644 --- a/tests/unit/test_langchain.py +++ b/tests/unit/test_langchain.py @@ -104,6 +104,38 @@ def test_chat_model_callback_exports_generation_span( } +def test_chat_model_generation_uses_model_reported_in_response( + langfuse_memory_client, get_span +): + response = ChatResult( + generations=[ + ChatGeneration( + message=AIMessage( + content="bonjour", + response_metadata={"model_name": "gpt-4o-2024-08-06"}, + ), + text="bonjour", + ) + ], + llm_output={"model_name": "azure.gpt-4o"}, + ) + + with patch.object(ChatOpenAI, "_generate", return_value=response): + handler = CallbackHandler() + ChatOpenAI(api_key="test", model="azure.gpt-4o").invoke( + [HumanMessage(content="hello")], + config={"callbacks": [handler]}, + ) + + langfuse_memory_client.flush() + generation_span = get_span("ChatOpenAI") + + assert ( + generation_span.attributes[LangfuseOtelSpanAttributes.OBSERVATION_MODEL] + == "gpt-4o-2024-08-06" + ) + + def test_chat_model_input_keeps_message_names( langfuse_memory_client, get_span, json_attr ):