diff --git a/demos/common/export_models/export_model.py b/demos/common/export_models/export_model.py index 137804cd20..9192b91449 100644 --- a/demos/common/export_models/export_model.py +++ b/demos/common/export_models/export_model.py @@ -54,8 +54,8 @@ def add_common_arguments(parser): parser_text.add_argument('--max_prompt_len', required=False, type=int, default=None, help='Sets NPU specific property for maximum number of tokens in the prompt. ' 'Not effective if target device is not NPU', dest='max_prompt_len') parser_text.add_argument('--prompt_lookup_decoding', action='store_true', help='Set pipeline to use prompt lookup decoding', dest='prompt_lookup_decoding') -parser_text.add_argument('--reasoning_parser', choices=["qwen3", "gptoss"], help='Set the type of the reasoning parser for reasoning content extraction', dest='reasoning_parser') -parser_text.add_argument('--tool_parser', choices=["llama3", "phi4", "hermes3", "mistral", "qwen3coder", "gptoss", "devstral", "lfm2"], help='Set the type of the tool parser for tool calls extraction', dest='tool_parser') +parser_text.add_argument('--reasoning_parser', choices=["qwen3", "gptoss", "lfm2", "gemma4"], help='Set the type of the reasoning parser for reasoning content extraction', dest='reasoning_parser') +parser_text.add_argument('--tool_parser', choices=["llama3", "phi4", "hermes3", "mistral", "qwen3coder", "gptoss", "devstral", "lfm2", "gemma4"], help='Set the type of the tool parser for tool calls extraction', dest='tool_parser') parser_text.add_argument('--enable_tool_guided_generation', action='store_true', help='Enables enforcing tool schema during generation. Requires setting tool_parser', dest='enable_tool_guided_generation') parser_embeddings_ov = subparsers.add_parser('embeddings_ov', help='export model for embeddings endpoint with directory structure aligned with OpenVINO tools') diff --git a/docs/llm/reference.md b/docs/llm/reference.md index eb9c82f635..bde4cc792c 100644 --- a/docs/llm/reference.md +++ b/docs/llm/reference.md @@ -289,8 +289,9 @@ __Tool parsers:__ __Reasoning parsers:__ - `qwen3` -- `gemma4` - `gptoss` +- `lfm2` +- `gemma4` #### Automatic parser detection diff --git a/docs/parameters.md b/docs/parameters.md index 102816dba4..7651cf0fa2 100644 --- a/docs/parameters.md +++ b/docs/parameters.md @@ -134,7 +134,7 @@ Task specific parameters for different tasks (text generation/image generation/e | `--max_prompt_len` | `integer` | Sets NPU specific property for maximum number of tokens in the prompt. | | `--kv_cache_precision` | `string` | Reduced kv cache precision to `u8` lowers the cache size consumption. Accepted values: `u8` or empty (default). | | `--model_distribution_policy` | `string` | TENSOR_PARALLEL distributes tensor to multiple sockets/devices and processes it in parallel. PIPELINE_PARALLEL distributes different tensors to process by each device. Accepted values: `TENSOR_PARALLEL`, `PIPELINE_PARALLEL` or empty (default). | -| `--reasoning_parser` | `string` | Type of parser to use for reasoning content extraction from model output. Auto-detected from chat template if not specified. Use `none` to explicitly disable. Supported: [qwen3, gptoss, gemma4] | +| `--reasoning_parser` | `string` | Type of parser to use for reasoning content extraction from model output. Auto-detected from chat template if not specified. Use `none` to explicitly disable. Supported: [qwen3, gptoss, lfm2, gemma4] | | `--tool_parser` | `string` | Type of parser to use for tool calls extraction from model output. Auto-detected from chat template if not specified. Use `none` to explicitly disable. Supported: [llama3, phi4, hermes3, mistral, qwen3coder, gptoss, devstral, lfm2, gemma4] | | `--enable_tool_guided_generation` | `bool` | Enables enforcing tool schema during generation. Requires setting response parser. Default: false. | diff --git a/extras/chat_template_examples/chat_template_lfm25.jinja b/extras/chat_template_examples/chat_template_lfm25.jinja new file mode 100644 index 0000000000..26acf01dfd --- /dev/null +++ b/extras/chat_template_examples/chat_template_lfm25.jinja @@ -0,0 +1,123 @@ +{#- + Modifications: + * Adding support for arguments as string so that minja does not detect the need to apply polyfills. This is dead path for OVMS as it always converts arguments to dict anyway. +#} +{{- bos_token -}} +{%- set preserve_thinking = preserve_thinking | default(false) -%} + +{%- macro format_arg_value(arg_value) -%} + {%- if arg_value is string -%} + {{- "'" + arg_value + "'" -}} + {%- elif arg_value is mapping -%} + {{- arg_value | tojson -}} + {%- else -%} + {{- arg_value | string -}} + {%- endif -%} +{%- endmacro -%} + +{%- macro parse_content(content) -%} + {%- if content is string -%} + {{- content -}} + {%- else -%} + {%- set _ns = namespace(result="") -%} + {%- for item in content -%} + {%- if item["type"] == "image" -%} + {%- set _ns.result = _ns.result + "" -%} + {%- elif item["type"] == "text" -%} + {%- set _ns.result = _ns.result + item["text"] -%} + {%- else -%} + {%- set _ns.result = _ns.result + item | tojson -%} + {%- endif -%} + {%- endfor -%} + {{- _ns.result -}} + {%- endif -%} +{%- endmacro -%} + +{%- macro render_tool_calls(tool_calls) -%} + {%- set tool_calls_ns = namespace(tool_calls=[]) -%} + {%- for tool_call in tool_calls -%} + {%- set func_name = tool_call["function"]["name"] -%} + {%- set func_args = tool_call["function"]["arguments"] -%} + {%- if func_args is string -%} + {%- set tool_calls_ns.tool_calls = tool_calls_ns.tool_calls + [func_name + "(" + func_args + ")"] -%} + {%- else -%} + {%- set args_ns = namespace(arg_strings=[]) -%} + {%- for arg_name, arg_value in func_args.items() -%} + {%- set args_ns.arg_strings = args_ns.arg_strings + [arg_name + "=" + format_arg_value(arg_value)] -%} + {%- endfor -%} + {%- set tool_calls_ns.tool_calls = tool_calls_ns.tool_calls + [func_name + "(" + (args_ns.arg_strings | join(", ")) + ")"] -%} + {%- endif -%} + {%- endfor -%} + {{- "<|tool_call_start|>[" + (tool_calls_ns.tool_calls | join(", ")) + "]<|tool_call_end|>" -}} +{%- endmacro -%} + +{%- set ns = namespace(system_prompt="", last_user_index=-1) -%} +{%- if messages[0]["role"] == "system" -%} + {%- if messages[0].get("content") -%} + {%- set ns.system_prompt = parse_content(messages[0]["content"]) -%} + {%- endif -%} + {%- set messages = messages[1:] -%} +{%- endif -%} +{%- if tools -%} + {%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: [" -%} + {%- for tool in tools -%} + {%- if tool is not string -%} + {%- set tool = tool | tojson -%} + {%- endif -%} + {%- set ns.system_prompt = ns.system_prompt + tool -%} + {%- if not loop.last -%} + {%- set ns.system_prompt = ns.system_prompt + ", " -%} + {%- endif -%} + {%- endfor -%} + {%- set ns.system_prompt = ns.system_prompt + "]" -%} +{%- endif -%} +{%- if ns.system_prompt -%} + {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}} +{%- endif -%} +{%- for message in messages -%} + {%- if message["role"] == "user" -%} + {%- set ns.last_user_index = loop.index0 -%} + {%- endif -%} +{%- endfor -%} +{%- for message in messages -%} + {{- "<|im_start|>" + message.role + "\n" -}} + {%- if message.role == "assistant" -%} + {%- generation -%} + {%- if message.thinking is defined and (preserve_thinking or loop.index0 > ns.last_user_index) -%} + {{- "" + message.thinking + "" -}} + {%- endif -%} + {%- set _cfm_tag = "CONTINUE_FINAL_MESSAGE_TAG " -%} + {%- set _has_cfm = false -%} + {%- if message.content is defined -%} + {%- set content = parse_content(message.content) -%} + {%- if not (preserve_thinking or loop.index0 > ns.last_user_index) -%} + {%- if "" in content -%} + {%- set content = content.split("")[-1] | trim -%} + {%- endif -%} + {%- endif -%} + {%- if message.tool_calls is defined and content.endswith(_cfm_tag) -%} + {%- set _has_cfm = true -%} + {%- set _trunc_len = (content | length) - (_cfm_tag | length) -%} + {{- content[:_trunc_len] -}} + {%- else -%} + {{- content -}} + {%- endif -%} + {%- endif -%} + {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 -%} + {{- render_tool_calls(message.tool_calls) -}} + {%- endif -%} + {%- if _has_cfm -%} + {{- _cfm_tag -}} + {%- endif -%} + {{- "<|im_end|>\n" -}} + {%- endgeneration -%} + {%- else %} + {%- if message.get("content") -%} + {{- parse_content(message["content"]) -}} + {%- endif -%} + {{- "<|im_end|>\n" -}} + {%- endif %} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{- "<|im_start|>assistant\n" -}} +{%- endif -%} \ No newline at end of file diff --git a/prepare_llm_models.sh b/prepare_llm_models.sh index 1ec7a61ba7..0bb0257580 100755 --- a/prepare_llm_models.sh +++ b/prepare_llm_models.sh @@ -39,6 +39,7 @@ MISTRAL_MODEL="mistralai/Mistral-7B-Instruct-v0.3" GPT_OSS_MODEL="openai/gpt-oss-20b" DEVSTRAL_MODEL="unsloth/Devstral-Small-2507" LFM2_MODEL="LiquidAI/LFM2-2.6B" +LFM25_MODEL="LiquidAI/LFM2.5-8B-A1B" GEMMA4_MODEL="OpenVINO/gemma-4-E4B-it-int4-ov" if [ "$(python3 -c 'import sys; print(sys.version_info[1])')" -le "8" ]; then echo "Prepare models with python > 3.8."; exit 1 ; fi @@ -208,6 +209,16 @@ if [ ! -f "$1/$LFM2_MODEL/$TOKENIZER_FILE" ]; then echo "[ERROR] Models file $1/$LFM2_MODEL/$TOKENIZER_FILE does not exist." exit 1 fi +if [ -f "$1/$LFM25_MODEL/$TOKENIZER_FILE" ]; then + echo "Models file $1/$LFM25_MODEL/$TOKENIZER_FILE exists. Skipping downloading models." +else + mkdir -p $1/$LFM25_MODEL + convert_tokenizer $LFM25_MODEL --with_detokenizer -o $1/$LFM25_MODEL +fi +if [ ! -f "$1/$LFM25_MODEL/$TOKENIZER_FILE" ]; then + echo "[ERROR] Models file $1/$LFM25_MODEL/$TOKENIZER_FILE does not exist." + exit 1 +fi if [ -f "$1/$GEMMA4_MODEL/$TOKENIZER_FILE" ]; then echo "Models file $1/$GEMMA4_MODEL/$TOKENIZER_FILE exists. Skipping downloading models." else diff --git a/spelling-whitelist.txt b/spelling-whitelist.txt index b458909bb3..81f083e9b5 100644 --- a/spelling-whitelist.txt +++ b/spelling-whitelist.txt @@ -29,6 +29,7 @@ demos/vlm_npu/README.md:157: mane ==> main, many, maine demos/vlm_npu/README.md:218: mane ==> main, many, maine demos/integration_with_OpenWebUI/README.md:423: Buildin ==> Building, Build in src/test/llm/output_parsers/lfm2_output_parser_test.cpp +src/test/llm/output_parsers/lfm25_output_parser_test.cpp windows_parse_tests.bat:35: seh ==> she windows_parse_tests.bat:119: SEH ==> SHE windows_parse_tests.bat:123: SEH ==> SHE diff --git a/src/llm/BUILD b/src/llm/BUILD index d6b91852b1..b3525b17ad 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -306,9 +306,9 @@ ovms_cc_library( ) ovms_cc_library( - name = "io_processing_lfm2_tool_parser", - hdrs = ["io_processing/lfm2/lfm2_tool_parser.hpp"], - srcs = ["io_processing/lfm2/lfm2_tool_parser.cpp"], + name = "io_processing_lfm2_utils", + hdrs = ["io_processing/lfm2/lfm2_utils.hpp"], + srcs = ["io_processing/lfm2/lfm2_utils.cpp"], deps = [ "@com_github_tencent_rapidjson//:rapidjson", "//src/port:rapidjson_document", @@ -320,6 +320,29 @@ ovms_cc_library( ], visibility = ["//visibility:public"], ) + +ovms_cc_library( + name = "io_processing_lfm2_tool_parser", + hdrs = ["io_processing/lfm2/lfm2_tool_parser.hpp"], + srcs = ["io_processing/lfm2/lfm2_tool_parser.cpp"], + deps = [ + ":io_processing_lfm2_utils", + ], + visibility = ["//visibility:public"], +) + + +ovms_cc_library( + name = "io_processing_lfm25_tool_parser", + hdrs = ["io_processing/lfm2/lfm25_tool_parser.hpp"], + srcs = ["io_processing/lfm2/lfm25_tool_parser.cpp"], + deps = [ + ":io_processing_lfm2_utils", + "//src:libovmslogging", + ], + visibility = ["//visibility:public"], +) + ovms_cc_library( name = "io_processing_gemma4_tool_parser", hdrs = ["io_processing/gemma4/gemma4_tool_parser.hpp", "io_processing/gemma4/gemma4_reasoning_parser.hpp"], @@ -353,6 +376,22 @@ ovms_cc_library( visibility = ["//visibility:public"], ) +ovms_cc_library( + name = "io_processing_lfm25_reasoning_parser", + hdrs = ["io_processing/lfm2/lfm25_reasoning_parser.hpp"], + srcs = ["io_processing/lfm2/lfm25_reasoning_parser.cpp"], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", + "//src/port:rapidjson_document", + "//src:libovmslogging", + "//src:libovmsstring_utils", + ":io_processing_utils", + ":io_processing_base_output_parser", + "//third_party:genai", + ], + visibility = ["//visibility:public"], +) + ovms_cc_library( # TODO split further so we don't have to recompile everything when changing one parser ... name = "output_parsers", hdrs = [ @@ -388,8 +427,10 @@ ovms_cc_library( # TODO split further so we don't have to recompile everything w ":io_processing_parser_config_validation", ":io_processing_qwen3coder_tool_parser", ":io_processing_lfm2_tool_parser", + ":io_processing_lfm25_tool_parser", ":io_processing_gemma4_tool_parser", ":io_processing_qwen3_reasoning_parser", + ":io_processing_lfm25_reasoning_parser", ":io_processing_utils", ":apis_tool_schema_wrapper", ], diff --git a/src/llm/io_processing/chat_template/analyzer.cpp b/src/llm/io_processing/chat_template/analyzer.cpp index c3d20aca09..29d4ab1801 100644 --- a/src/llm/io_processing/chat_template/analyzer.cpp +++ b/src/llm/io_processing/chat_template/analyzer.cpp @@ -56,13 +56,15 @@ ChatTemplateAnalysisResult ChatTemplateAnalyzer::analyze(const std::string& temp return result; } - // TODO: It does not work for LFM2, but only for LFM2.5? Both use the same parsers // LFM2 detection - if (contains(templateSource, "<|assistant_tool_call|>") || contains(templateSource, "<|tool_call_start|>")) { + if (contains(templateSource, "<|assistant_tool_call|>") || contains(templateSource, "<|tool_call_start|>") || contains(templateSource, "keep_past_thinking")) { result.detectedToolParser = "lfm2"; result.caps.supportsToolCalls = true; + if (contains(templateSource, "message.thinking")) { + result.caps.missnamedReasoningField = "thinking"; + result.detectedReasoningParser = "lfm2"; + } return result; - // TODO: Support reasoning after Pawel adds reasoning parser for it } // Phi-4 detection — uses "functools[" marker for tool calls diff --git a/src/llm/io_processing/chat_template/caps.hpp b/src/llm/io_processing/chat_template/caps.hpp index 5904fc6f08..4fd2885487 100644 --- a/src/llm/io_processing/chat_template/caps.hpp +++ b/src/llm/io_processing/chat_template/caps.hpp @@ -25,13 +25,16 @@ struct ChatTemplateCaps { // Some templates require tool_call arguments to be a dict/object rather than a stringified JSON. bool requiresObjectArguments = false; + std::string missnamedReasoningField = ""; + bool needsWorkarounds() const { - return requiresObjectArguments; + return requiresObjectArguments || !missnamedReasoningField.empty(); } std::string toString() const { return std::string("supportsToolCalls=") + (supportsToolCalls ? "true" : "false") + - ", requiresObjectArguments=" + (requiresObjectArguments ? "true" : "false"); + ", requiresObjectArguments=" + (requiresObjectArguments ? "true" : "false") + + ", missnamedReasoningField=" + missnamedReasoningField; } }; diff --git a/src/llm/io_processing/chat_template/probe.cpp b/src/llm/io_processing/chat_template/probe.cpp index 136d5405f2..fe4a73dab2 100644 --- a/src/llm/io_processing/chat_template/probe.cpp +++ b/src/llm/io_processing/chat_template/probe.cpp @@ -84,9 +84,12 @@ static const std::string PROBE_STR_ARGS_MSG = static const std::string PROBE_OBJ_ARGS_MSG = R"({"role":"assistant","content":"","tool_calls":[{"id":"call_0_ab","type":"function","function":{"name":"probe_fn","arguments":{")" + PROBE_NEEDLE + R"(":"val"}}}]})"; +static const std::string PROBE_REASONING_MSG = + R"({"role":"assistant","reasoning_content":")" + PROBE_NEEDLE + R"("})"; + // Analyze dry-run probe outputs and update caps accordingly. // Returns false if the template silently failed (tool calls not supported). -static bool analyzeProbeResults(bool strOk, const std::string& strOut, +static bool analyzeProbeToolArgumentResults(bool strOk, const std::string& strOut, bool objOk, const std::string& objOut, ChatTemplateCaps& caps) { auto rendersNativeArgs = [](const std::string& output) -> bool { @@ -129,6 +132,56 @@ static bool analyzeProbeResults(bool strOk, const std::string& strOut, return true; } +bool probeChatTemplateReasoning(ov::genai::Tokenizer& tokenizer, ChatTemplateCaps& caps) { + if (tokenizer.get_chat_template().empty()) { + return true; + } + if (!caps.needsWorkarounds() || caps.missnamedReasoningField.empty()) { + return true; + } + + try { + ov::genai::ChatHistory history; + history.push_back(ov::genai::JsonContainer::from_json_string(R"({"role":"user","content":"Hello"})")); + history.push_back(ov::genai::JsonContainer::from_json_string(PROBE_REASONING_MSG)); + + std::string output = tokenizer.apply_chat_template(history, true); + + if (output.find(PROBE_NEEDLE) != std::string::npos) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Reasoning probe: standard 'reasoning_content' field supported"); + caps.missnamedReasoningField.clear(); + return true; + } + + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Reasoning probe: 'reasoning_content' field silently ignored"); + } catch (const std::exception& e) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Reasoning probe: 'reasoning_content' not supported: {}", e.what()); + } catch (...) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Reasoning probe: 'reasoning_content' not supported (unknown exception)"); + } + + auto field = caps.missnamedReasoningField; + std::string altMsg = R"({"role":"assistant",")" + field + R"(":")" + PROBE_NEEDLE + R"("})"; + try { + ov::genai::ChatHistory history; + history.push_back(ov::genai::JsonContainer::from_json_string(R"({"role":"user","content":"Hello"})")); + history.push_back(ov::genai::JsonContainer::from_json_string(altMsg)); + + std::string output = tokenizer.apply_chat_template(history, true); + + if (output.find(PROBE_NEEDLE) != std::string::npos) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Reasoning probe: template uses '{}' instead of 'reasoning_content'", field); + return true; + } + } catch (...) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Reasoning probe: '{}' not supported", field); + return false; + } + + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Reasoning probe: no reasoning field supported by template"); + return false; +} + static std::pair renderProbeMessage(ov::genai::Tokenizer& tokenizer, const std::string& assistantMsg) { try { ov::genai::ChatHistory history; @@ -163,7 +216,7 @@ bool probeChatTemplateCapsMinja(ov::genai::Tokenizer& tokenizer, ChatTemplateCap auto [strOk, strOut] = strArgsFuture.get(); auto [objOk, objOut] = objArgsFuture.get(); - return analyzeProbeResults(strOk, strOut, objOk, objOut, caps); + return analyzeProbeToolArgumentResults(strOk, strOut, objOk, objOut, caps); } #if (PYTHON_DISABLE == 0) @@ -197,7 +250,7 @@ bool probeChatTemplateCapsJinja(PyJinjaTemplateProcessor& templateProcessor, Cha return false; } - return analyzeProbeResults(strOk, strOut, objOk, objOut, caps); + return analyzeProbeToolArgumentResults(strOk, strOut, objOk, objOut, caps); } #endif diff --git a/src/llm/io_processing/chat_template/probe.hpp b/src/llm/io_processing/chat_template/probe.hpp index a1ee4ad126..416dab94fb 100644 --- a/src/llm/io_processing/chat_template/probe.hpp +++ b/src/llm/io_processing/chat_template/probe.hpp @@ -38,6 +38,8 @@ bool probeChatTemplateBasicRenderMinja(ov::genai::Tokenizer& tokenizer); // Returns false if minja silently fails to render (template unsupported). bool probeChatTemplateCapsMinja(ov::genai::Tokenizer& tokenizer, ChatTemplateCaps& caps); +bool probeChatTemplateReasoning(ov::genai::Tokenizer& tokenizer, ChatTemplateCaps& caps); + #if (PYTHON_DISABLE == 0) class PyJinjaTemplateProcessor; diff --git a/src/llm/io_processing/input_processors/chat_template_adapter.cpp b/src/llm/io_processing/input_processors/chat_template_adapter.cpp index c0ffe29a0c..f1f0f82590 100644 --- a/src/llm/io_processing/input_processors/chat_template_adapter.cpp +++ b/src/llm/io_processing/input_processors/chat_template_adapter.cpp @@ -60,11 +60,25 @@ void funcArgsToObjectHistory(ov::genai::ChatHistory& chatHistory) { } } +void injectReasoningIntoMissnamedSection(ov::genai::ChatHistory& chatHistory, const std::string& templateReasoningFieldName) { + for (size_t msgIdx = 0; msgIdx < chatHistory.size(); ++msgIdx) { + auto message = chatHistory[msgIdx]; + if (!message.contains("reasoning_content") || !message["reasoning_content"].is_string()) { + continue; + } + + message[templateReasoningFieldName.c_str()] = message["reasoning_content"].get_string(); + } +} + void applyToHistory(const ChatTemplateCaps& caps, ov::genai::ChatHistory& chatHistory) { SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Applying chat template adaptations: {}", caps.toString()); if (caps.requiresObjectArguments) { funcArgsToObjectHistory(chatHistory); } + if (!caps.missnamedReasoningField.empty()) { + injectReasoningIntoMissnamedSection(chatHistory, caps.missnamedReasoningField); + } } } // namespace chat_template_adapter diff --git a/src/llm/io_processing/lfm2/lfm25_reasoning_parser.cpp b/src/llm/io_processing/lfm2/lfm25_reasoning_parser.cpp new file mode 100644 index 0000000000..8d49e384dd --- /dev/null +++ b/src/llm/io_processing/lfm2/lfm25_reasoning_parser.cpp @@ -0,0 +1,72 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include + +#include "src/port/rapidjson_document.hpp" + +#include "../../../logging.hpp" +#include "lfm25_reasoning_parser.hpp" +#include "../utils.hpp" + +namespace ovms { +void Lfm25ReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { + auto startReasoningIt = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningStartTokenId); + auto endReasoningIt = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningEndTokenId); + + if (startReasoningIt == generatedTokens.end() || endReasoningIt == generatedTokens.end() || startReasoningIt >= endReasoningIt) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Lfm25ReasoningParser: Reasoning start or end token not found in the generated tokens, or in wrong order. Start token found: {}, End token found: {}, Start position: {}, End position: {}", + startReasoningIt != generatedTokens.end(), endReasoningIt != generatedTokens.end(), std::distance(generatedTokens.begin(), startReasoningIt), std::distance(generatedTokens.begin(), endReasoningIt)); + return; + } + + auto startPos = std::distance(generatedTokens.begin(), startReasoningIt); + auto endPos = std::distance(generatedTokens.begin(), endReasoningIt); + + std::string reasoningContent = tokenizer.decode(std::vector(startPos + generatedTokens.begin() + 1, endPos + generatedTokens.begin()), ov::genai::skip_special_tokens(true)); + + parsedOutput.reasoning = reasoningContent; + + std::string contentWithoutReasoning = tokenizer.decode(std::vector(generatedTokens.begin() + endPos + 1, generatedTokens.end()), ov::genai::skip_special_tokens(true)); // content MUST never appear before reasoning + parsedOutput.content = contentWithoutReasoning; +} + +std::optional Lfm25ReasoningParser::parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) { + if (tokens.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty tokens for Lfm25ReasoningParser"); + return std::nullopt; + } + + if (std::find(tokens.begin(), tokens.end(), reasoningStartTokenId) != tokens.end() || + std::find(tokens.begin(), tokens.end(), reasoningEndTokenId) != tokens.end()) { + return std::nullopt; + } else { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + writer.String("delta"); + writer.StartObject(); + writer.String("reasoning_content"); + writer.String(chunk.c_str()); + writer.EndObject(); + writer.EndObject(); + rapidjson::Document doc; + doc.Parse(buffer.GetString()); + return doc; + } +} +} // namespace ovms diff --git a/src/llm/io_processing/lfm2/lfm25_reasoning_parser.hpp b/src/llm/io_processing/lfm2/lfm25_reasoning_parser.hpp new file mode 100644 index 0000000000..afc52c7f56 --- /dev/null +++ b/src/llm/io_processing/lfm2/lfm25_reasoning_parser.hpp @@ -0,0 +1,54 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include "../base_output_parser.hpp" +#include +#include + +namespace ovms { +class Lfm25ReasoningParser : public BaseOutputParser { +protected: + const std::string parsingStartTag = ""; + const std::string parsingEndTag = ""; + + const int64_t reasoningStartTokenId = 124901; // + const int64_t reasoningEndTokenId = 124902; // + +public: + Lfm25ReasoningParser() = delete; + explicit Lfm25ReasoningParser(ov::genai::Tokenizer& tokenizer) : + BaseOutputParser(tokenizer) {} + + void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + const std::vector& getParsingStartTags() const override { + static const std::vector parsingStartTags{this->parsingStartTag}; + return parsingStartTags; + } + const std::vector& getSpecialParsingStartTags() const override { + static const std::vector specialParsingStartTags{}; + return specialParsingStartTags; + } + const std::string& getParsingEndTag() const override { + return parsingEndTag; + } + + // It may be removed after changing logic in Lfm2ToolParser to use tokens in streaming instead of chunk content, both tool parser and reasoning parser need to have the same value for this function + bool requiresStreamingWithSpecialTokens() const override { + return true; + } +}; +} // namespace ovms diff --git a/src/llm/io_processing/lfm2/lfm25_tool_parser.cpp b/src/llm/io_processing/lfm2/lfm25_tool_parser.cpp new file mode 100644 index 0000000000..d651ed359f --- /dev/null +++ b/src/llm/io_processing/lfm2/lfm25_tool_parser.cpp @@ -0,0 +1,108 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "lfm25_tool_parser.hpp" + +namespace ovms { + +const std::string Lfm25ToolParser::TOOL_CALL_START_TAG = "<|tool_call_start|>"; +const std::string Lfm25ToolParser::TOOL_CALL_END_TAG = "<|tool_call_end|>"; + +const int64_t Lfm25ToolParser::toolCallStartTokenId = 124905; // <|tool_call_start|> +const int64_t Lfm25ToolParser::toolCallEndTokenId = 124906; // <|tool_call_end|> +const int64_t Lfm25ToolParser::reasoningStartTokenId = 124901; // +const int64_t Lfm25ToolParser::reasoningEndTokenId = 124902; // + +bool Lfm25ToolParser::parseNewContent() { + switch (this->currentState) { + case State::Content: { + return parseInContentState(this->streamingContent, this->streamingPosition, this->currentState, this->tagIds); + } + case State::ToolCallStarted: { + auto wasParsedCorrectly = parseInToolCallState(this->streamingContent, this->toolCall, this->streamingPosition, this->currentState); + if (wasParsedCorrectly) { + this->toolCallIndex++; + } + return wasParsedCorrectly; + } + case State::ToolCallParameters: { + return parseInToolCallParametersState(this->streamingContent, this->toolCall, this->streamingPosition, this->currentState); + } + case State::ToolCallEnded: { + return parseInToolCallEndedState(this->streamingContent, this->streamingPosition, this->currentState, TOOL_CALL_END_TAG); + } + case State::AfterToolCall: + break; + } + return false; +} + +std::optional Lfm25ToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { + if (chunk.empty()) { + return std::nullopt; + } + + this->streamingContent += chunk; + + if (parseNewContent()) { + if (this->currentState == State::ToolCallParameters) { + return BaseOutputParser::wrapFirstDelta(this->toolCall.name, this->toolCallIndex); + } + if (this->currentState == State::ToolCallEnded) { + return wrapDeltaArgs(this->toolCall.arguments, this->toolCallIndex); + } + if (this->currentState == State::Content) { + size_t contentEnd = this->streamingContent.find(TOOL_CALL_START_TAG, this->streamingPosition); + std::string content; + if (contentEnd != std::string::npos) { + content = this->streamingContent.substr(this->streamingPosition, contentEnd - this->streamingPosition); + } else { + content = this->streamingContent.substr(this->streamingPosition); + } + this->streamingPosition += content.size(); + cutEOSFromContent(content); + + if (!content.empty()) { + return wrapDeltaContent(content); + } + } + if (this->currentState == State::AfterToolCall) { + this->currentState = State::Content; + } + } + + if (finishReason != ov::genai::GenerationFinishReason::NONE) { + if ((this->currentState == State::ToolCallParameters || this->currentState == State::ToolCallEnded) && !this->toolCall.arguments.empty()) { + return wrapDeltaArgs(this->toolCall.arguments, this->toolCallIndex); + } + + if (this->currentState == State::Content && this->streamingPosition < this->streamingContent.size()) { + auto content = this->streamingContent.substr(this->streamingPosition); + this->streamingPosition += content.size(); + cutEOSFromContent(content); + + if (!content.empty()) { + return wrapDeltaContent(content); + } + } + } + + return std::nullopt; +} + +void Lfm25ToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { + parseUnaryResponse(parsedOutput, generatedTokens, tokenizer, this->tagIds); +} +} // namespace ovms diff --git a/src/llm/io_processing/lfm2/lfm25_tool_parser.hpp b/src/llm/io_processing/lfm2/lfm25_tool_parser.hpp new file mode 100644 index 0000000000..abcf288dcb --- /dev/null +++ b/src/llm/io_processing/lfm2/lfm25_tool_parser.hpp @@ -0,0 +1,72 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include +#include +#include "lfm2_utils.hpp" + +namespace ovms { +class Lfm25ToolParser : public BaseOutputParser { +public: + static const std::string TOOL_CALL_START_TAG; + static const std::string TOOL_CALL_END_TAG; + + static const int64_t toolCallStartTokenId; + static const int64_t toolCallEndTokenId; + static const int64_t reasoningStartTokenId; + static const int64_t reasoningEndTokenId; + + Lfm25ToolParser() = delete; + explicit Lfm25ToolParser(ov::genai::Tokenizer& tokenizer) : + BaseOutputParser(tokenizer) {} + + void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + const std::vector& getParsingStartTags() const override { + static const std::vector parsingStartTags = {TOOL_CALL_START_TAG}; + return parsingStartTags; + } + + const std::vector& getSpecialParsingStartTags() const override { + static const std::vector beginningOnlyTags = {}; + return beginningOnlyTags; + } + + const std::vector& getSpecialTagsToErase() const override { + static const std::vector tagsToErase = {EOS_TOKEN_STR}; + return tagsToErase; + } + + const std::string& getParsingEndTag() const override { + return TOOL_CALL_END_TAG; + } + + bool requiresStreamingWithSpecialTokens() const override { + return true; + } + +private: + std::string streamingContent; + size_t streamingPosition{0}; + State currentState{State::Content}; + ToolCall toolCall; + TagIds tagIds{TOOL_CALL_START_TAG, TOOL_CALL_END_TAG, toolCallStartTokenId, toolCallEndTokenId, reasoningStartTokenId, reasoningEndTokenId}; + + int toolCallIndex{TOOL_CALL_INDEX_START}; + + bool parseNewContent(); +}; +} // namespace ovms diff --git a/src/llm/io_processing/lfm2/lfm2_tool_parser.cpp b/src/llm/io_processing/lfm2/lfm2_tool_parser.cpp index 056a5bead9..3e13a2b681 100644 --- a/src/llm/io_processing/lfm2/lfm2_tool_parser.cpp +++ b/src/llm/io_processing/lfm2/lfm2_tool_parser.cpp @@ -14,293 +14,32 @@ // limitations under the License. //***************************************************************************** #include "lfm2_tool_parser.hpp" -#include "../utils.hpp" -#include "../../../logging.hpp" -#include "../../../stringutils.hpp" -#include "rapidjson/error/en.h" -#include -#include -#include namespace ovms { const std::string Lfm2ToolParser::TOOL_CALL_START_TAG = "<|tool_call_start|>"; const std::string Lfm2ToolParser::TOOL_CALL_END_TAG = "<|tool_call_end|>"; -const std::string Lfm2ToolParser::EOS_TOKEN_STR = "<|im_end|>"; -const std::string Lfm2ToolParser::TOOL_LIST_START_INDICATOR = "["; -const std::string Lfm2ToolParser::TOOL_LIST_END_INDICATOR = "]"; -const std::string Lfm2ToolParser::TOOL_ARGS_START_INDICATOR = "("; -const std::string Lfm2ToolParser::TOOL_ARGS_END_INDICATOR = ")"; -const std::string Lfm2ToolParser::TOOL_SEPARATOR_STR = ", "; - -const int64_t Lfm2ToolParser::botTokenId = 10; -const int64_t Lfm2ToolParser::eotTokenId = 11; - -std::string Lfm2ToolParser::parseArrayParameter(std::string argumentStr) { - int quoteDepth = 0; - - for (size_t i = 1; i < argumentStr.size() - 1; ++i) { - if (argumentStr[i] != '\'') { - continue; - } - - bool isLastElement = (i == argumentStr.size() - 2); - bool isFollowedByComma = !isLastElement && argumentStr[i + 1] == ','; - - if (quoteDepth == 0) { - argumentStr[i] = '"'; - quoteDepth++; - } else if (quoteDepth > 0 && (isFollowedByComma || isLastElement)) { - argumentStr[i] = '"'; - quoteDepth--; - } - } - - return argumentStr; -} - -std::string Lfm2ToolParser::parseObjectParameter(std::string argumentStr) { - int quoteDepth = 0; - - for (size_t i = 1; i < argumentStr.size() - 1; ++i) { - if (argumentStr[i] != '\'') { - continue; - } - - bool isLastElement = (i == argumentStr.size() - 2); - bool isFollowedByComma = !isLastElement && argumentStr[i + 1] == ','; - bool isFollowedByColon = !isLastElement && argumentStr[i + 1] == ':'; - - if (quoteDepth == 0) { - argumentStr[i] = '"'; - quoteDepth++; - } else if (quoteDepth > 0 && (isFollowedByComma || isLastElement || isFollowedByColon)) { - argumentStr[i] = '"'; - quoteDepth--; - } - } - - return argumentStr; -} - -std::string Lfm2ToolParser::normalizeArgStr(const std::string& arg) { - if (arg.empty()) { - return arg; - } - - std::string normalized = arg; - trim(normalized); - std::string lower = normalized; - std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); - - if (lower == "true" || lower == "false" || lower == "null") { - return lower; - } - - const char first = normalized.front(); - const char last = normalized.back(); - if (first == '{' && last == '}') { - normalized = parseObjectParameter(normalized); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument contains is an object, replaced single quotes with double quotes for JSON parsing. Modified string: {}", normalized); - } - - if (first == '[' && last == ']') { - normalized = parseArrayParameter(normalized); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is an array, normalized quotes for JSON parsing. Modified string: {}", normalized); - } - - if ((first == '\'' && last == '\'')) { - normalized[0] = '"'; - normalized[normalized.size() - 1] = '"'; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is enclosed in quotes, replaced outer quotes with double quotes for JSON parsing. Modified string: {}", normalized); - } - - rapidjson::Document tempDoc; - rapidjson::Value finalValue; - tempDoc.Parse(normalized.c_str()); - if (tempDoc.HasParseError()) { - auto errorCode = tempDoc.GetParseError(); - auto errorMessage = rapidjson::GetParseError_En(errorCode); - size_t errorOffset = tempDoc.GetErrorOffset(); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Failed to parse argument string as JSON. Argument string: {}, Error: {} Offset: {}", normalized, errorMessage, errorOffset); - - if (first == '\"' && last == '\"') { - normalized = normalized.substr(1, normalized.size() - 2); - } - finalValue.SetString(normalized.c_str(), static_cast(normalized.size()), tempDoc.GetAllocator()); - } else { - finalValue.CopyFrom(tempDoc, tempDoc.GetAllocator()); - } - - { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - finalValue.Accept(writer); - normalized = buffer.GetString(); - } - - return normalized; -} - -void Lfm2ToolParser::writeArgumentToWriter(const std::string& arg, rapidjson::Writer& writer) { - std::string normalized = normalizeArgStr(arg); - - rapidjson::Document doc; - doc.Parse(normalized.c_str()); - - rapidjson::Value& argumentDoc = doc; - writeArgumentOfAnyType(argumentDoc, writer); -} - -Lfm2ToolParser::Argument Lfm2ToolParser::parseSingleArgument(const std::string& argumentStr) { - Lfm2ToolParser::Argument argument; - - size_t equalPos = argumentStr.find('='); - if (equalPos != std::string::npos) { - argument.name = argumentStr.substr(0, equalPos); - argument.value = argumentStr.substr(equalPos + 1); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed argument - name: {}, value: {}", argument.name, argument.value); - } else { - argument.name = argumentStr; - argument.value = ""; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument string: {} does not contain '=', setting name as entire string and value as empty", argumentStr); - } - return argument; -} - -std::vector Lfm2ToolParser::parseArguments(const std::string& argumentsStr) { - std::vector args; - std::vector parsedArgs; - - size_t argPos = 0; - while (argPos < argumentsStr.length()) { - size_t commaPos = findInStringRespectingSpecialChars(argumentsStr, TOOL_SEPARATOR_STR, argPos); - if (commaPos == std::string::npos) { - auto remainingStr = argumentsStr.substr(argPos); - args.push_back(remainingStr); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "No more commas found, adding remaining argument string: {}", remainingStr); - break; - } - auto argStr = argumentsStr.substr(argPos, commaPos - argPos); - args.push_back(argStr); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed argument string: {}", argStr); - argPos = commaPos + TOOL_SEPARATOR_STR.length(); - } - - for (const std::string& arg : args) { - parsedArgs.push_back(parseSingleArgument(arg)); - } - return parsedArgs; -} - -bool Lfm2ToolParser::parseInContentState() { - size_t toolCallStartTagPos = this->streamingContent.find(TOOL_CALL_START_TAG, this->streamingPosition); - size_t toolCallEndTagPos = this->streamingContent.find(TOOL_CALL_END_TAG, this->streamingPosition); - if (toolCallEndTagPos != std::string::npos && toolCallStartTagPos == std::string::npos) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool call at position: {}", toolCallEndTagPos); - this->streamingPosition = toolCallEndTagPos + TOOL_CALL_END_TAG.length(); - return false; - } - if (toolCallStartTagPos != std::string::npos) { - if (toolCallStartTagPos > this->streamingPosition) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Content found before tool call start tag at position: {}", toolCallStartTagPos); - return true; - } - this->streamingPosition = toolCallStartTagPos + TOOL_CALL_START_TAG.length(); - this->currentState = State::ToolCallStarted; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected start of tool call at position: {}", toolCallStartTagPos); - return false; - } - - return true; -} - -bool Lfm2ToolParser::parseInToolCallState() { - size_t toolListStartPos = this->streamingContent.find(TOOL_LIST_START_INDICATOR, this->streamingPosition); - size_t argsPos = this->streamingContent.find(TOOL_ARGS_START_INDICATOR, this->streamingPosition); - - if (toolListStartPos != std::string::npos) { - this->streamingPosition = toolListStartPos + TOOL_LIST_START_INDICATOR.length(); - } - - if (argsPos == std::string::npos) { - return false; - } - - std::string toolName = this->streamingContent.substr(this->streamingPosition, argsPos - this->streamingPosition); - trim(toolName); - this->toolCall = ToolCall{generateRandomId(), toolName, ""}; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool name: {}", toolName); - this->streamingPosition = argsPos + TOOL_ARGS_START_INDICATOR.length(); - this->currentState = State::ToolCallParameters; - this->toolCallIndex++; - return true; -} - -bool Lfm2ToolParser::parseToolCallParametersState() { - size_t pos = findInStringRespectingSpecialChars(this->streamingContent, TOOL_ARGS_END_INDICATOR, this->streamingPosition); - if (pos == std::string::npos) { - return false; - } - std::string argumentsStr = this->streamingContent.substr(this->streamingPosition, pos - this->streamingPosition); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed arguments string: {}", argumentsStr); - std::vector arguments = parseArguments(argumentsStr); - - rapidjson::Document argsDoc(rapidjson::kObjectType); - rapidjson::StringBuffer sb; - rapidjson::Writer argsWriter(sb); - argsWriter.StartObject(); - - for (const Argument& argument : arguments) { - argsWriter.Key(argument.name.c_str()); - writeArgumentToWriter(argument.value, argsWriter); - } - - argsWriter.EndObject(); - this->toolCall.arguments = sb.GetString(); - this->currentState = State::ToolCallEnded; - this->streamingPosition = pos + TOOL_ARGS_END_INDICATOR.length(); - - return true; -} - -bool Lfm2ToolParser::parseInToolCallEndedState() { - size_t pos = this->streamingContent.find(TOOL_LIST_END_INDICATOR, this->streamingPosition); - size_t toolSeparatorPos = this->streamingContent.find(TOOL_SEPARATOR_STR, this->streamingPosition); - size_t toolCallEndTagPos = this->streamingContent.find(TOOL_CALL_END_TAG, this->streamingPosition); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Current state: ToolCallEnded. Streaming content from current position: {}", this->streamingContent.substr(this->streamingPosition)); - if (pos == std::string::npos && toolSeparatorPos == std::string::npos && toolCallEndTagPos == std::string::npos) { - return false; - } else if (toolSeparatorPos != std::string::npos && toolSeparatorPos < pos) { - this->streamingPosition = toolSeparatorPos + TOOL_SEPARATOR_STR.length(); - this->currentState = State::ToolCallStarted; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected separator between tool calls at position: {}, expecting another tool call to start", toolSeparatorPos); - } else if (toolCallEndTagPos != std::string::npos) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool call at position: {}", toolCallEndTagPos); - this->streamingPosition = toolCallEndTagPos + TOOL_CALL_END_TAG.length(); - this->currentState = State::AfterToolCall; - } else { - this->streamingPosition = pos + TOOL_LIST_END_INDICATOR.length(); - this->currentState = State::AfterToolCall; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool list at position: {}, returning to content state", pos); - } - return true; -} +const int64_t Lfm2ToolParser::toolCallStartTokenId = 10; // <|tool_call_start|> +const int64_t Lfm2ToolParser::toolCallEndTokenId = 11; // <|tool_call_end|> bool Lfm2ToolParser::parseNewContent() { switch (this->currentState) { case State::Content: { - return parseInContentState(); + return parseInContentState(this->streamingContent, this->streamingPosition, this->currentState, this->tagIds); } case State::ToolCallStarted: { - return parseInToolCallState(); + auto wasParsedCorrectly = parseInToolCallState(this->streamingContent, this->toolCall, this->streamingPosition, this->currentState); + if (wasParsedCorrectly) { + this->toolCallIndex++; + } + return wasParsedCorrectly; } case State::ToolCallParameters: { - return parseToolCallParametersState(); + return parseInToolCallParametersState(this->streamingContent, this->toolCall, this->streamingPosition, this->currentState); } case State::ToolCallEnded: { - return parseInToolCallEndedState(); + return parseInToolCallEndedState(this->streamingContent, this->streamingPosition, this->currentState, TOOL_CALL_END_TAG); } case State::AfterToolCall: break; @@ -308,28 +47,6 @@ bool Lfm2ToolParser::parseNewContent() { return false; } -rapidjson::Document Lfm2ToolParser::wrapDeltaContent(const std::string& content) { - rapidjson::Document doc(rapidjson::kObjectType); - rapidjson::Value deltaObj(rapidjson::kObjectType); - deltaObj.AddMember("content", rapidjson::Value(content.c_str(), doc.GetAllocator()), doc.GetAllocator()); - doc.AddMember("delta", deltaObj, doc.GetAllocator()); - return doc; -} - -rapidjson::Document Lfm2ToolParser::wrapDeltaArgs(const std::string& argsStr) { - rapidjson::Document doc(rapidjson::kObjectType); - doc.AddMember("arguments", rapidjson::Value(argsStr.c_str(), doc.GetAllocator()), doc.GetAllocator()); - - return BaseOutputParser::wrapDelta(doc, this->toolCallIndex); -} - -void Lfm2ToolParser::cutEOSFromContent(std::string& content) { - size_t eosPos = content.find(EOS_TOKEN_STR); - if (eosPos != std::string::npos) { - content = content.substr(0, eosPos); - } -} - std::optional Lfm2ToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { if (chunk.empty()) { return std::nullopt; @@ -342,7 +59,7 @@ std::optional Lfm2ToolParser::parseChunk(const std::string& return BaseOutputParser::wrapFirstDelta(this->toolCall.name, this->toolCallIndex); } if (this->currentState == State::ToolCallEnded) { - return wrapDeltaArgs(this->toolCall.arguments); + return wrapDeltaArgs(this->toolCall.arguments, this->toolCallIndex); } if (this->currentState == State::Content) { size_t contentEnd = this->streamingContent.find(TOOL_CALL_START_TAG, this->streamingPosition); @@ -366,7 +83,7 @@ std::optional Lfm2ToolParser::parseChunk(const std::string& if (finishReason != ov::genai::GenerationFinishReason::NONE) { if ((this->currentState == State::ToolCallParameters || this->currentState == State::ToolCallEnded) && !this->toolCall.arguments.empty()) { - return wrapDeltaArgs(this->toolCall.arguments); + return wrapDeltaArgs(this->toolCall.arguments, this->toolCallIndex); } if (this->currentState == State::Content && this->streamingPosition < this->streamingContent.size()) { @@ -383,99 +100,7 @@ std::optional Lfm2ToolParser::parseChunk(const std::string& return std::nullopt; } -bool Lfm2ToolParser::parseSingleToolCall(const std::string& toolStr, ToolCall& toolCall) { - size_t argsPos = toolStr.find(TOOL_ARGS_START_INDICATOR); - if (argsPos != std::string::npos) { - std::string toolName = toolStr.substr(0, argsPos); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool name: {}", toolName); - - int argsStrLen = toolStr.length() - argsPos - TOOL_ARGS_START_INDICATOR.length() - TOOL_ARGS_END_INDICATOR.length(); - std::string argsStr = toolStr.substr(argsPos + TOOL_ARGS_START_INDICATOR.length(), argsStrLen); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed args string: {}", argsStr); - std::vector arguments = parseArguments(argsStr); - - toolCall.name = toolName; - rapidjson::Document argsDoc(rapidjson::kObjectType); - rapidjson::StringBuffer sb; - rapidjson::Writer argsWriter(sb); - argsWriter.StartObject(); - for (const Lfm2ToolParser::Argument& argument : arguments) { - argsWriter.Key(argument.name.c_str()); - writeArgumentToWriter(argument.value, argsWriter); - } - argsWriter.EndObject(); - toolCall.arguments = sb.GetString(); - toolCall.id = generateRandomId(); - return true; - } - return false; -} - void Lfm2ToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - std::vector tools; - std::vector> toolCallPositions; - size_t pos = 0; - int mainGuard = 0; - - while (pos != std::string::npos && mainGuard < MAX_TOOL_CALLS) { - size_t start, end; - auto it = std::find(generatedTokens.begin() + pos, generatedTokens.end(), botTokenId); - if (it != generatedTokens.end()) { - start = std::distance(generatedTokens.begin(), it); - } else { - break; - } - auto itArgs = std::find(generatedTokens.begin() + start, generatedTokens.end(), eotTokenId); - if (itArgs != generatedTokens.end()) { - end = std::distance(generatedTokens.begin(), itArgs); - } else { - break; - } - - std::string toolListStr = tokenizer.decode(std::vector(generatedTokens.begin() + start + 1, generatedTokens.begin() + end), ov::AnyMap{ov::genai::skip_special_tokens(false)}); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool list string: {}", toolListStr); - int toolGuard = 0; - toolListStr = toolListStr.substr(TOOL_LIST_START_INDICATOR.length(), toolListStr.length() - TOOL_LIST_START_INDICATOR.length() - TOOL_LIST_END_INDICATOR.length()); - - while (!toolListStr.empty() && toolGuard < MAX_TOOLS_PER_CALL) { - size_t toolEndPos = findInStringRespectingSpecialChars(toolListStr, TOOL_ARGS_END_INDICATOR, 0); - std::string singleTool; - if (toolEndPos != std::string::npos) { - singleTool = toolListStr.substr(0, toolEndPos + TOOL_ARGS_END_INDICATOR.length()); - if (toolEndPos + TOOL_ARGS_END_INDICATOR.length() < toolListStr.length()) { - toolListStr = toolListStr.substr(toolEndPos + TOOL_ARGS_END_INDICATOR.length() + TOOL_SEPARATOR_STR.length()); - } else { - toolListStr.clear(); - } - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed single tool string {}", singleTool); - } - - if (!singleTool.empty()) { - tools.push_back(singleTool); - } - toolGuard++; - } - mainGuard++; - - pos = end; - toolCallPositions.emplace_back(start, end); - } - - for (const std::string& tool : tools) { - ToolCall toolCall; - auto wasToolCallParsed = parseSingleToolCall(tool, toolCall); - if (wasToolCallParsed) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool call - name: {}, args: {}", toolCall.name, toolCall.arguments); - parsedOutput.toolCalls.push_back(toolCall); - } else { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Failed to parse tool call from string: {}", tool); - } - } - - std::vector contentWithoutToolCalls = generatedTokens; - for (auto it = toolCallPositions.rbegin(); it != toolCallPositions.rend(); ++it) { - contentWithoutToolCalls.erase(contentWithoutToolCalls.begin() + it->first, contentWithoutToolCalls.begin() + it->second + 1); - } - parsedOutput.content = tokenizer.decode(contentWithoutToolCalls, ov::AnyMap{ov::genai::skip_special_tokens(true)}); + parseUnaryResponse(parsedOutput, generatedTokens, tokenizer, this->tagIds); } } // namespace ovms diff --git a/src/llm/io_processing/lfm2/lfm2_tool_parser.hpp b/src/llm/io_processing/lfm2/lfm2_tool_parser.hpp index cd56634eb4..228616d33d 100644 --- a/src/llm/io_processing/lfm2/lfm2_tool_parser.hpp +++ b/src/llm/io_processing/lfm2/lfm2_tool_parser.hpp @@ -17,39 +17,19 @@ #include #include #include "src/llm/io_processing/base_output_parser.hpp" +#include "../../../logging.hpp" +#include "./lfm2_utils.hpp" namespace ovms { class Lfm2ToolParser : public BaseOutputParser { protected: static const std::string TOOL_CALL_START_TAG; static const std::string TOOL_CALL_END_TAG; - static const std::string EOS_TOKEN_STR; - static const std::string TOOL_LIST_START_INDICATOR; - static const std::string TOOL_LIST_END_INDICATOR; - static const std::string TOOL_ARGS_START_INDICATOR; - static const std::string TOOL_ARGS_END_INDICATOR; - static const std::string TOOL_SEPARATOR_STR; - - static const int64_t botTokenId; - static const int64_t eotTokenId; - - static constexpr size_t MAX_TOOL_CALLS = 100; - static constexpr size_t MAX_TOOLS_PER_CALL = 100; - static constexpr int TOOL_CALL_INDEX_START = -1; - enum class State { - Content, // Content -> ToolCallStarted (on TOOL_CALL_START_TAG) - ToolCallStarted, // ToolCallStarted -> ToolCallParameters (on TOOL_ARGS_START_INDICATOR, emits name) - ToolCallParameters, // ToolCallParameters -> ToolCallEnded (on TOOL_ARGS_END_INDICATOR, emits args) - ToolCallEnded, // ToolCallEnded -> ToolCallStarted (on separator) | AfterToolCall (on end tag/list end) - AfterToolCall // AfterToolCall -> Content - }; + static const int64_t toolCallStartTokenId; + static const int64_t toolCallEndTokenId; public: - struct Argument { - std::string name; - std::string value; - }; Lfm2ToolParser() = delete; explicit Lfm2ToolParser(ov::genai::Tokenizer& tokenizer) : BaseOutputParser(tokenizer) {} @@ -79,32 +59,15 @@ class Lfm2ToolParser : public BaseOutputParser { return true; } - static std::string normalizeArgStr(const std::string& arg); - static std::string parseArrayParameter(std::string argumentStr); - static std::string parseObjectParameter(std::string argumentStr); - private: - void writeArgumentToWriter(const std::string& arg, rapidjson::Writer& writer); - - Argument parseSingleArgument(const std::string& argumentStr); - std::vector parseArguments(const std::string& argumentsStr); - void cutEOSFromContent(std::string& content); - - bool parseSingleToolCall(const std::string& toolStr, ToolCall& toolCall); - bool parseNewContent(); - bool parseInContentState(); - bool parseInToolCallState(); - bool parseToolCallParametersState(); - bool parseInToolCallEndedState(); - - rapidjson::Document wrapDeltaContent(const std::string& content); - rapidjson::Document wrapDeltaArgs(const std::string& argsStr); - std::string streamingContent; size_t streamingPosition{0}; State currentState{State::Content}; ToolCall toolCall; + TagIds tagIds{TOOL_CALL_START_TAG, TOOL_CALL_END_TAG, toolCallStartTokenId, toolCallEndTokenId}; int toolCallIndex{TOOL_CALL_INDEX_START}; + + bool parseNewContent(); }; } // namespace ovms diff --git a/src/llm/io_processing/lfm2/lfm2_utils.cpp b/src/llm/io_processing/lfm2/lfm2_utils.cpp new file mode 100644 index 0000000000..331d442a45 --- /dev/null +++ b/src/llm/io_processing/lfm2/lfm2_utils.cpp @@ -0,0 +1,412 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "lfm2_utils.hpp" +#include "../utils.hpp" +#include "../../../logging.hpp" +#include "../../../stringutils.hpp" +#include "rapidjson/error/en.h" +#include +#include +#include + +namespace ovms { +const std::string TOOL_LIST_START_INDICATOR = "["; +const std::string TOOL_LIST_END_INDICATOR = "]"; +const std::string TOOL_ARGS_START_INDICATOR = "("; +const std::string TOOL_ARGS_END_INDICATOR = ")"; +const std::string TOOL_SEPARATOR_STR = ", "; +const std::string EOS_TOKEN_STR = "<|im_end|>"; + +const int TOOL_CALL_INDEX_START = -1; + +std::string parseArrayParameter(std::string argumentStr) { + int quoteDepth = 0; + + for (size_t i = 1; i < argumentStr.size() - 1; ++i) { + if (argumentStr[i] != '\'') { + continue; + } + + bool isLastElement = (i == argumentStr.size() - 2); + bool isFollowedByComma = !isLastElement && argumentStr[i + 1] == ','; + + if (quoteDepth == 0) { + argumentStr[i] = '"'; + quoteDepth++; + } else if (quoteDepth > 0 && (isFollowedByComma || isLastElement)) { + argumentStr[i] = '"'; + quoteDepth--; + } + } + + return argumentStr; +} + +std::string parseObjectParameter(std::string argumentStr) { + int quoteDepth = 0; + + for (size_t i = 1; i < argumentStr.size() - 1; ++i) { + if (argumentStr[i] != '\'') { + continue; + } + + bool isLastElement = (i == argumentStr.size() - 2); + bool isFollowedByComma = !isLastElement && argumentStr[i + 1] == ','; + bool isFollowedByColon = !isLastElement && argumentStr[i + 1] == ':'; + + if (quoteDepth == 0) { + argumentStr[i] = '"'; + quoteDepth++; + } else if (quoteDepth > 0 && (isFollowedByComma || isLastElement || isFollowedByColon)) { + argumentStr[i] = '"'; + quoteDepth--; + } + } + + return argumentStr; +} + +std::string normalizeArgStr(const std::string& arg) { + if (arg.empty()) { + return arg; + } + + std::string normalized = arg; + trim(normalized); + std::string lower = normalized; + std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + + if (lower == "true" || lower == "false" || lower == "null") { + return lower; + } + + const char first = normalized.front(); + const char last = normalized.back(); + if (first == '{' && last == '}') { + normalized = parseObjectParameter(normalized); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument contains is an object, replaced single quotes with double quotes for JSON parsing. Modified string: {}", normalized); + } + + if (first == '[' && last == ']') { + normalized = parseArrayParameter(normalized); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is an array, normalized quotes for JSON parsing. Modified string: {}", normalized); + } + + if ((first == '\'' && last == '\'')) { + normalized[0] = '"'; + normalized[normalized.size() - 1] = '"'; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is enclosed in quotes, replaced outer quotes with double quotes for JSON parsing. Modified string: {}", normalized); + } + + rapidjson::Document tempDoc; + rapidjson::Value finalValue; + tempDoc.Parse(normalized.c_str()); + if (tempDoc.HasParseError()) { + auto errorCode = tempDoc.GetParseError(); + auto errorMessage = rapidjson::GetParseError_En(errorCode); + size_t errorOffset = tempDoc.GetErrorOffset(); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Failed to parse argument string as JSON. Argument string: {}, Error: {} Offset: {}", normalized, errorMessage, errorOffset); + + if (first == '\"' && last == '\"') { + normalized = normalized.substr(1, normalized.size() - 2); + } + finalValue.SetString(normalized.c_str(), static_cast(normalized.size()), tempDoc.GetAllocator()); + } else { + finalValue.CopyFrom(tempDoc, tempDoc.GetAllocator()); + } + + { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + finalValue.Accept(writer); + normalized = buffer.GetString(); + } + + return normalized; +} + +void writeArgumentToWriter(const std::string& arg, rapidjson::Writer& writer) { + std::string normalized = normalizeArgStr(arg); + + rapidjson::Document doc; + doc.Parse(normalized.c_str()); + + rapidjson::Value& argumentDoc = doc; + writeArgumentOfAnyType(argumentDoc, writer); +} + +Argument parseSingleArgument(const std::string& argumentStr) { + Argument argument; + + size_t equalPos = argumentStr.find('='); + if (equalPos != std::string::npos) { + argument.name = argumentStr.substr(0, equalPos); + argument.value = argumentStr.substr(equalPos + 1); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed argument - name: {}, value: {}", argument.name, argument.value); + } else { + argument.name = argumentStr; + argument.value = ""; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument string: {} does not contain '=', setting name as entire string and value as empty", argumentStr); + } + return argument; +} + +std::vector parseArguments(const std::string& argumentsStr) { + std::vector args; + std::vector parsedArgs; + + size_t argPos = 0; + while (argPos < argumentsStr.length()) { + size_t commaPos = findInStringRespectingSpecialChars(argumentsStr, TOOL_SEPARATOR_STR, argPos); + if (commaPos == std::string::npos) { + auto remainingStr = argumentsStr.substr(argPos); + args.push_back(remainingStr); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "No more commas found, adding remaining argument string: {}", remainingStr); + break; + } + auto argStr = argumentsStr.substr(argPos, commaPos - argPos); + args.push_back(argStr); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed argument string: {}", argStr); + argPos = commaPos + TOOL_SEPARATOR_STR.length(); + } + + for (const std::string& arg : args) { + parsedArgs.push_back(parseSingleArgument(arg)); + } + return parsedArgs; +} + +bool parseInContentState(const std::string& streamingContent, size_t& streamingPosition, State& currentState, const TagIds& tagIds) { + size_t toolCallStartTagPos = streamingContent.find(tagIds.toolCallStartTag, streamingPosition); + size_t toolCallEndTagPos = streamingContent.find(tagIds.toolCallEndTag, streamingPosition); + if (toolCallEndTagPos != std::string::npos && toolCallStartTagPos == std::string::npos) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool call at position: {}", toolCallEndTagPos); + streamingPosition = toolCallEndTagPos + tagIds.toolCallEndTag.length(); + return false; + } + if (toolCallStartTagPos != std::string::npos) { + if (toolCallStartTagPos > streamingPosition) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Content found before tool call start tag at position: {}", toolCallStartTagPos); + return true; + } + currentState = State::ToolCallStarted; + streamingPosition = toolCallStartTagPos + tagIds.toolCallStartTag.length(); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected start of tool call at position: {}", toolCallStartTagPos); + return false; + } + + return true; +} + +bool parseInToolCallState(const std::string& streamingContent, ToolCall& toolCall, size_t& streamingPosition, State& currentState) { + size_t toolListStartPos = streamingContent.find(TOOL_LIST_START_INDICATOR, streamingPosition); + size_t argsPos = streamingContent.find(TOOL_ARGS_START_INDICATOR, streamingPosition); + + if (toolListStartPos != std::string::npos) { + streamingPosition = toolListStartPos + TOOL_LIST_START_INDICATOR.length(); + } + + if (argsPos == std::string::npos) { + return false; + } + + std::string toolName = streamingContent.substr(streamingPosition, argsPos - streamingPosition); + trim(toolName); + toolCall = ToolCall{generateRandomId(), toolName, ""}; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool name: {}", toolName); + streamingPosition = argsPos + TOOL_ARGS_START_INDICATOR.length(); + currentState = State::ToolCallParameters; + return true; +} + +bool parseInToolCallParametersState(const std::string& streamingContent, ToolCall& toolCall, size_t& streamingPosition, State& currentState) { + size_t pos = findInStringRespectingSpecialChars(streamingContent, TOOL_ARGS_END_INDICATOR, streamingPosition); + if (pos == std::string::npos) { + return false; + } + std::string argumentsStr = streamingContent.substr(streamingPosition, pos - streamingPosition); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed arguments string: {}", argumentsStr); + std::vector arguments = parseArguments(argumentsStr); + + rapidjson::Document argsDoc(rapidjson::kObjectType); + rapidjson::StringBuffer sb; + rapidjson::Writer argsWriter(sb); + argsWriter.StartObject(); + + for (const Argument& argument : arguments) { + argsWriter.Key(argument.name.c_str()); + writeArgumentToWriter(argument.value, argsWriter); + } + + argsWriter.EndObject(); + toolCall.arguments = sb.GetString(); + currentState = State::ToolCallEnded; + streamingPosition = pos + TOOL_ARGS_END_INDICATOR.length(); + + return true; +} + +bool parseInToolCallEndedState(const std::string& streamingContent, size_t& streamingPosition, State& currentState, const std::string& toolCallEndTag) { + size_t pos = streamingContent.find(TOOL_LIST_END_INDICATOR, streamingPosition); + size_t toolSeparatorPos = streamingContent.find(TOOL_SEPARATOR_STR, streamingPosition); + size_t toolCallEndTagPos = streamingContent.find(toolCallEndTag, streamingPosition); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Current state: ToolCallEnded. Streaming content from current position: {}", streamingContent.substr(streamingPosition)); + if (pos == std::string::npos && toolSeparatorPos == std::string::npos && toolCallEndTagPos == std::string::npos) { + return false; + } else if (toolSeparatorPos != std::string::npos && toolSeparatorPos < pos) { + streamingPosition = toolSeparatorPos + TOOL_SEPARATOR_STR.length(); + currentState = State::ToolCallStarted; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected separator between tool calls at position: {}, expecting another tool call to start", toolSeparatorPos); + } else if (toolCallEndTagPos != std::string::npos) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool call at position: {}", toolCallEndTagPos); + streamingPosition = toolCallEndTagPos + toolCallEndTag.length(); + currentState = State::AfterToolCall; + } else { + streamingPosition = pos + TOOL_LIST_END_INDICATOR.length(); + currentState = State::AfterToolCall; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool list at position: {}, returning to content state", pos); + } + return true; +} + +rapidjson::Document wrapDeltaContent(const std::string& content) { + rapidjson::Document doc(rapidjson::kObjectType); + rapidjson::Value deltaObj(rapidjson::kObjectType); + deltaObj.AddMember("content", rapidjson::Value(content.c_str(), doc.GetAllocator()), doc.GetAllocator()); + doc.AddMember("delta", deltaObj, doc.GetAllocator()); + return doc; +} + +rapidjson::Document wrapDeltaArgs(const std::string& argsStr, int toolCallIndex) { + rapidjson::Document doc(rapidjson::kObjectType); + doc.AddMember("arguments", rapidjson::Value(argsStr.c_str(), doc.GetAllocator()), doc.GetAllocator()); + + return BaseOutputParser::wrapDelta(doc, toolCallIndex); +} + +void cutEOSFromContent(std::string& content) { + size_t eosPos = content.find(EOS_TOKEN_STR); + if (eosPos != std::string::npos) { + content = content.substr(0, eosPos); + } +} + +bool parseSingleToolCall(const std::string& toolStr, ToolCall& toolCall) { + size_t argsPos = toolStr.find(TOOL_ARGS_START_INDICATOR); + if (argsPos != std::string::npos) { + std::string toolName = toolStr.substr(0, argsPos); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool name: {}", toolName); + + int argsStrLen = toolStr.length() - argsPos - TOOL_ARGS_START_INDICATOR.length() - TOOL_ARGS_END_INDICATOR.length(); + std::string argsStr = toolStr.substr(argsPos + TOOL_ARGS_START_INDICATOR.length(), argsStrLen); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed args string: {}", argsStr); + std::vector arguments = parseArguments(argsStr); + + toolCall.name = toolName; + rapidjson::Document argsDoc(rapidjson::kObjectType); + rapidjson::StringBuffer sb; + rapidjson::Writer argsWriter(sb); + argsWriter.StartObject(); + for (const Argument& argument : arguments) { + argsWriter.Key(argument.name.c_str()); + writeArgumentToWriter(argument.value, argsWriter); + } + argsWriter.EndObject(); + toolCall.arguments = sb.GetString(); + toolCall.id = generateRandomId(); + return true; + } + return false; +} + +void parseUnaryResponse(ParsedOutput& parsedOutput, const std::vector& generatedTokens, ov::genai::Tokenizer& tokenizer, const TagIds& tagIds) { + std::vector tools; + std::vector> toolCallPositions; + size_t pos = 0; + + while (pos != std::string::npos) { + size_t start, end; + auto it = std::find(generatedTokens.begin() + pos, generatedTokens.end(), tagIds.toolCallStartTokenId); + if (it != generatedTokens.end()) { + start = std::distance(generatedTokens.begin(), it); + } else { + break; + } + auto itArgs = std::find(generatedTokens.begin() + start, generatedTokens.end(), tagIds.toolCallEndTokenId); + if (itArgs != generatedTokens.end()) { + end = std::distance(generatedTokens.begin(), itArgs); + } else { + break; + } + + std::string toolListStr = tokenizer.decode(std::vector(generatedTokens.begin() + start + 1, generatedTokens.begin() + end), ov::AnyMap{ov::genai::skip_special_tokens(false)}); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool list string: {}", toolListStr); + toolListStr = toolListStr.substr(TOOL_LIST_START_INDICATOR.length(), toolListStr.length() - TOOL_LIST_START_INDICATOR.length() - TOOL_LIST_END_INDICATOR.length()); + + while (!toolListStr.empty()) { + size_t toolEndPos = findInStringRespectingSpecialChars(toolListStr, TOOL_ARGS_END_INDICATOR, 0); + std::string singleTool; + if (toolEndPos != std::string::npos) { + singleTool = toolListStr.substr(0, toolEndPos + TOOL_ARGS_END_INDICATOR.length()); + if (toolEndPos + TOOL_ARGS_END_INDICATOR.length() < toolListStr.length()) { + toolListStr = toolListStr.substr(toolEndPos + TOOL_ARGS_END_INDICATOR.length() + TOOL_SEPARATOR_STR.length()); + } else { + toolListStr.clear(); + } + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed single tool string {}", singleTool); + } else { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "No more tool calls found in tool list string: {}", toolListStr); + break; + } + + if (!singleTool.empty()) { + tools.push_back(singleTool); + } + } + pos = end; + toolCallPositions.emplace_back(start, end); + } + + for (const std::string& tool : tools) { + ToolCall toolCall; + auto wasToolCallParsed = parseSingleToolCall(tool, toolCall); + if (wasToolCallParsed) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool call - name: {}, args: {}", toolCall.name, toolCall.arguments); + parsedOutput.toolCalls.push_back(toolCall); + } else { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Failed to parse tool call from string: {}", tool); + } + } + + std::vector contentWithoutToolCalls = generatedTokens; + for (auto it = toolCallPositions.rbegin(); it != toolCallPositions.rend(); ++it) { + contentWithoutToolCalls.erase(contentWithoutToolCalls.begin() + it->first, contentWithoutToolCalls.begin() + it->second + 1); + } + if (tagIds.reasoningEndTokenId.has_value() && tagIds.reasoningStartTokenId.has_value()) { + auto reasoningEndIt = std::find(contentWithoutToolCalls.begin(), contentWithoutToolCalls.end(), tagIds.reasoningEndTokenId.value()); + if (reasoningEndIt != contentWithoutToolCalls.end()) { + contentWithoutToolCalls.erase(contentWithoutToolCalls.begin(), reasoningEndIt + 1); + } else { + auto reasoningStartIt = std::find(contentWithoutToolCalls.begin(), contentWithoutToolCalls.end(), tagIds.reasoningStartTokenId.value()); + if (reasoningStartIt != contentWithoutToolCalls.end()) { + contentWithoutToolCalls.erase(reasoningStartIt, contentWithoutToolCalls.end()); + } + } + } + + parsedOutput.content = tokenizer.decode(contentWithoutToolCalls, ov::AnyMap{ov::genai::skip_special_tokens(true)}); +} +} // namespace ovms diff --git a/src/llm/io_processing/lfm2/lfm2_utils.hpp b/src/llm/io_processing/lfm2/lfm2_utils.hpp new file mode 100644 index 0000000000..e6bc49761d --- /dev/null +++ b/src/llm/io_processing/lfm2/lfm2_utils.hpp @@ -0,0 +1,69 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include +#include +#include "src/llm/io_processing/base_output_parser.hpp" + +namespace ovms { + +extern const std::string TOOL_LIST_START_INDICATOR; +extern const std::string TOOL_LIST_END_INDICATOR; +extern const std::string TOOL_ARGS_START_INDICATOR; +extern const std::string TOOL_ARGS_END_INDICATOR; +extern const std::string TOOL_SEPARATOR_STR; +extern const std::string EOS_TOKEN_STR; +extern const int TOOL_CALL_INDEX_START; + +struct Argument { + std::string name; + std::string value; +}; + +enum class State { + Content, + ToolCallStarted, + ToolCallParameters, + ToolCallEnded, + AfterToolCall +}; + +struct TagIds { + std::string toolCallStartTag; + std::string toolCallEndTag; + int64_t toolCallStartTokenId; + int64_t toolCallEndTokenId; + std::optional reasoningStartTokenId = std::nullopt; + std::optional reasoningEndTokenId = std::nullopt; +}; + +std::string parseArrayParameter(std::string argumentStr); +std::string parseObjectParameter(std::string argumentStr); +std::string normalizeArgStr(const std::string& arg); +void writeArgumentToWriter(const std::string& arg, rapidjson::Writer& writer); +Argument parseSingleArgument(const std::string& argumentStr); +std::vector parseArguments(const std::string& argumentsStr); +bool parseInContentState(const std::string& streamingContent, size_t& streamingPosition, State& currentState, const TagIds& tagIds); +bool parseInToolCallState(const std::string& streamingContent, ToolCall& toolCall, size_t& streamingPosition, State& currentState); +bool parseInToolCallParametersState(const std::string& streamingContent, ToolCall& toolCall, size_t& streamingPosition, State& currentState); +bool parseInToolCallEndedState(const std::string& streamingContent, size_t& streamingPosition, State& currentState, const std::string& toolCallEndTag); +rapidjson::Document wrapDeltaContent(const std::string& content); +rapidjson::Document wrapDeltaArgs(const std::string& argsStr, int toolCallIndex); +void cutEOSFromContent(std::string& content); +bool parseSingleToolCall(const std::string& toolStr, ToolCall& toolCall); +void parseUnaryResponse(ParsedOutput& parsedOutput, const std::vector& generatedTokens, ov::genai::Tokenizer& tokenizer, const TagIds& tagIds); + +} // namespace ovms diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index e676f2bea6..679ed8425d 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -32,6 +32,8 @@ #include "gemma4/gemma4_reasoning_parser.hpp" #include "gptoss/reasoning_parser.hpp" #include "lfm2/lfm2_tool_parser.hpp" +#include "lfm2/lfm25_tool_parser.hpp" +#include "lfm2/lfm25_reasoning_parser.hpp" #include "gemma4/gemma4_tool_parser.hpp" namespace ovms { @@ -193,7 +195,16 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to } else if (toolParserName == "devstral") { toolParser = std::make_unique(tokenizer, toolNameSchemaMap); } else if (toolParserName == "lfm2") { - toolParser = std::make_unique(tokenizer); + auto vocab = tokenizer.get_vocab(); + auto token = vocab.find(Lfm25ToolParser::TOOL_CALL_START_TAG); + auto tokenId = token != vocab.end() ? token->second : -1; + if (tokenId == Lfm25ToolParser::toolCallStartTokenId) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Using Lfm25ToolParser for tool parsing"); + toolParser = std::make_unique(tokenizer); + } else { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Using Lfm2ToolParser for tool parsing"); + toolParser = std::make_unique(tokenizer); + } } else if (toolParserName == "gemma4") { toolParser = std::make_unique(tokenizer); } else if (!toolParserName.empty()) { @@ -207,11 +218,14 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "gptoss") { reasoningParser = std::make_unique(tokenizer); + } else if (reasoningParserName == "lfm2") { + reasoningParser = std::make_unique(tokenizer); } else if (!reasoningParserName.empty()) { throw std::runtime_error("Unsupported reasoning parser: \"" + reasoningParserName + "\". Supported reasoning parsers are: " + getSupportedReasoningParserNamesAsString()); } + // TODO: To be considered: If we still need this check after introduction of OvmsTextStreamer. if (toolParser && reasoningParser) { if (toolParser->requiresStreamingWithSpecialTokens() != reasoningParser->requiresStreamingWithSpecialTokens()) { throw std::runtime_error("Cannot use tool parser " + toolParserName + " with reasoning parser " + reasoningParserName + diff --git a/src/llm/io_processing/parser_config_validation.cpp b/src/llm/io_processing/parser_config_validation.cpp index 31d645cadc..cba275f7e5 100644 --- a/src/llm/io_processing/parser_config_validation.cpp +++ b/src/llm/io_processing/parser_config_validation.cpp @@ -41,6 +41,7 @@ const std::vector& getSupportedReasoningParserNames() { "qwen3", "gemma4", "gptoss", + "lfm2", }; return names; } diff --git a/src/llm/servable_initializer.cpp b/src/llm/servable_initializer.cpp index e2bcac9134..d0053939a0 100644 --- a/src/llm/servable_initializer.cpp +++ b/src/llm/servable_initializer.cpp @@ -82,6 +82,10 @@ static void probeServableChatTemplateCaps(std::shared_ptrtokenizer, properties->chatTemplateCaps)) { SPDLOG_LOGGER_WARN(llm_calculator_logger, "Minja cannot render this template's tool calls correctly"); } + + if (!properties->reasoningParserName.empty() && !probeChatTemplateReasoning(properties->tokenizer, properties->chatTemplateCaps)) { + SPDLOG_LOGGER_WARN(llm_calculator_logger, "Chat template does not support reasoning_content field"); + } } void GenAiServableInitializer::loadChatTemplate(std::shared_ptr properties, const std::string& chatTemplateDirectory) { diff --git a/src/test/llm/chat_template_analyzer_test.cpp b/src/test/llm/chat_template_analyzer_test.cpp index e889cff35e..56f18f5458 100644 --- a/src/test/llm/chat_template_analyzer_test.cpp +++ b/src/test/llm/chat_template_analyzer_test.cpp @@ -94,8 +94,10 @@ TEST_F(ChatTemplateAnalyzerTest, detectsLfm25) { auto result = ChatTemplateAnalyzer::analyze(tmpl); ASSERT_TRUE(result.detectedToolParser.has_value()); EXPECT_EQ(result.detectedToolParser.value(), "lfm2"); - EXPECT_FALSE(result.detectedReasoningParser.has_value()); + ASSERT_TRUE(result.detectedReasoningParser.has_value()); + EXPECT_EQ(result.detectedReasoningParser.value(), "lfm2"); EXPECT_TRUE(result.caps.supportsToolCalls); + EXPECT_EQ(result.caps.missnamedReasoningField, "thinking"); } // --- Phi-4 --- @@ -204,4 +206,5 @@ TEST_F(ChatTemplateAnalyzerTest, defaultCapsValues) { ChatTemplateCaps caps; EXPECT_FALSE(caps.supportsToolCalls); EXPECT_FALSE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); } diff --git a/src/test/llm/chat_template_end_to_end_jinja_test.cpp b/src/test/llm/chat_template_end_to_end_jinja_test.cpp index 3ec42b0f94..50d0130448 100644 --- a/src/test/llm/chat_template_end_to_end_jinja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_jinja_test.cpp @@ -140,13 +140,23 @@ class ChatTemplateEndToEndJinjaTest : public TestWithTempDir { ASSERT_NE(servable->getProperties()->templateProcessor.chatTemplate, nullptr) << "Failed to load Python Jinja template processor"; - // Step 3: Probe tool caps using Python Jinja (same function used in production) + // Step 3a: Probe tool caps using Python Jinja (same function used in production) if (!probeChatTemplateCapsJinja(servable->getProperties()->templateProcessor, caps)) { std::cout << "=== Jinja Probe FAILED: silent failure detected ===" << std::endl; } + // Step 3b: Probe reasoning caps using Python Jinja (same function used in production) + { + ov::genai::Tokenizer probeTokenizer(tokenizerModelPath); + probeTokenizer.set_chat_template(chatTemplate); + if (!probeChatTemplateReasoning(probeTokenizer, caps)) { + std::cout << "=== Jinja Reasoning Probe FAILED: silent failure detected ===" << std::endl; + } + } + std::cout << "=== After Probe ===" << std::endl; std::cout << " requiresObjectArguments: " << caps.requiresObjectArguments << std::endl; + std::cout << " missnamedReasoningField: " << (caps.missnamedReasoningField.empty() ? "(none)" : caps.missnamedReasoningField) << std::endl; // Step 4: Apply workarounds to chat history chat_template_adapter::applyToHistory(caps, chatHistory); @@ -194,6 +204,7 @@ TEST_F(ChatTemplateEndToEndJinjaTest, GptOss_ToolCallWithStringArgs) { EXPECT_TRUE(caps.supportsToolCalls); EXPECT_FALSE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); std::string expectedOutput = R"(<|start|>user<|message|>What's the weather in Paris?<|end|><|start|>assistant to=functions.get_weather <|channel|>commentary json<|message|>{"location":"Paris","unit":"celsius"}<|end|><|start|>assistant)"; EXPECT_NE(appliedOutput.find(expectedOutput), std::string::npos) << appliedOutput; @@ -222,6 +233,7 @@ TEST_F(ChatTemplateEndToEndJinjaTest, Qwen36_ToolCallWithStringArgs) { EXPECT_TRUE(caps.supportsToolCalls); EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); std::string expectedOutput = R"(<|im_start|>user What's the weather in Paris?<|im_end|> @@ -269,6 +281,7 @@ TEST_F(ChatTemplateEndToEndJinjaTest, Gemma4_ToolCallWithStringArgs) { EXPECT_TRUE(caps.supportsToolCalls); EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); std::string expectedOutput = R"(<|turn>user What's the weather in Paris? @@ -301,6 +314,7 @@ TEST_F(ChatTemplateEndToEndJinjaTest, Qwen3Coder_ToolCallWithStringArgs) { EXPECT_TRUE(caps.supportsToolCalls); EXPECT_FALSE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); std::string expectedOutput = R"(<|im_start|>user What's the weather in Paris?<|im_end|> @@ -343,6 +357,7 @@ TEST_F(ChatTemplateEndToEndJinjaTest, Phi4Mini_ToolCallWithStringArgs) { EXPECT_TRUE(caps.supportsToolCalls); EXPECT_FALSE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); std::string expectedOutput = R"(<|system|> You are a helpful assistant.<|end|><|user|>What's the weather in Paris?<|end|><|assistant|>{"name": "get_weather", "arguments": {"location":"Paris","unit":"celsius"}}<|end|><|assistant|>)"; @@ -372,6 +387,7 @@ TEST_F(ChatTemplateEndToEndJinjaTest, Qwen3_ToolCallWithStringArgs) { EXPECT_TRUE(caps.supportsToolCalls); EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); std::string expectedOutput = R"(<|im_start|>user What's the weather in Paris?<|im_end|> @@ -410,13 +426,14 @@ TEST_F(ChatTemplateEndToEndJinjaTest, Mistral7B_ToolCallWithStringArgs) { EXPECT_TRUE(caps.supportsToolCalls); EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); std::string expectedOutput = R"([INST] What's the weather in Paris?[/INST][TOOL_CALLS] [{"name": "get_weather", "arguments": {"location": "Paris", "unit": "celsius"}, "id": "abc123def"}])"; EXPECT_EQ(appliedOutput, expectedOutput); } // ============================================================================= -// TODO: Implement assertions, where to take lfm2 deployments steps from? +// LFM2 does not render tool calls in current chat template, therefore the output is the same as if there were no tool calls. // ============================================================================= TEST_F(ChatTemplateEndToEndJinjaTest, LFM2_ToolCallWithStringArgs) { chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_lfm2.jinja"); @@ -429,7 +446,85 @@ TEST_F(ChatTemplateEndToEndJinjaTest, LFM2_ToolCallWithStringArgs) { run(); - // TODO: Implement assertions, where to take lfm2 deployments steps from? + ASSERT_FALSE(exceptionThrownDuringApplication); + + ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "lfm2"); + ASSERT_FALSE(analysisResult.detectedReasoningParser.has_value()); + + EXPECT_TRUE(caps.supportsToolCalls); + EXPECT_FALSE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); + + std::string expectedOutput = R"(<|im_start|>user +What's the weather in Paris?<|im_end|> +<|im_start|>assistant +<|im_end|> +<|im_start|>assistant +)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + +TEST_F(ChatTemplateEndToEndJinjaTest, LFM25_ToolCallWithStringArgsAndReasoning) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_lfm25.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load lfm2.5 template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant", "reasoning_content":"Here is some reasoning content","content":"","tool_calls":[{"id":"call_abc123","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"}}]})")); + + run(); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "lfm2"); + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "lfm2"); + + EXPECT_TRUE(caps.supportsToolCalls); + EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_EQ(caps.missnamedReasoningField, "thinking"); + + std::string expectedOutput = R"(<|im_start|>user +What's the weather in Paris?<|im_end|> +<|im_start|>assistant +Here is some reasoning content<|tool_call_start|>[get_weather(location='Paris', unit='celsius')]<|tool_call_end|><|im_end|> +<|im_start|>assistant +)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + +TEST_F(ChatTemplateEndToEndJinjaTest, LFM25_ReasoningAndEmptyToolCalls) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_lfm25.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load lfm2.5 template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant", "reasoning_content":"Here is some reasoning content","content":"","tool_calls":[]})")); + + run(); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "lfm2"); + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "lfm2"); + + EXPECT_TRUE(caps.supportsToolCalls); + EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_EQ(caps.missnamedReasoningField, "thinking"); + + std::string expectedOutput = R"(<|im_start|>user +What's the weather in Paris?<|im_end|> +<|im_start|>assistant +Here is some reasoning content<|im_end|> +<|im_start|>assistant +)"; + EXPECT_EQ(appliedOutput, expectedOutput); } // ============================================================================= @@ -450,9 +545,10 @@ TEST_F(ChatTemplateEndToEndJinjaTest, LFM25_ToolCallWithStringArgs) { ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); EXPECT_EQ(analysisResult.detectedToolParser.value(), "lfm2"); - ASSERT_FALSE(analysisResult.detectedReasoningParser.has_value()); - // TODO(przepeck): change once we have reasoning tool parser for lfm2.5 + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "lfm2"); + EXPECT_EQ(caps.missnamedReasoningField, "thinking"); EXPECT_TRUE(caps.supportsToolCalls); EXPECT_TRUE(caps.requiresObjectArguments); @@ -487,6 +583,7 @@ TEST_F(ChatTemplateEndToEndJinjaTest, Qwen3VL_ToolCallWithStringArgs) { EXPECT_TRUE(caps.supportsToolCalls); EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); std::string expectedOutput = R"(<|im_start|>user What's the weather in Paris?<|im_end|> @@ -521,6 +618,7 @@ TEST_F(ChatTemplateEndToEndJinjaTest, Qwen3_30B_ToolCallWithStringArgs) { EXPECT_TRUE(caps.supportsToolCalls); EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); std::string expectedOutput = R"(<|im_start|>user What's the weather in Paris?<|im_end|> diff --git a/src/test/llm/chat_template_end_to_end_minja_test.cpp b/src/test/llm/chat_template_end_to_end_minja_test.cpp index 83e057275a..1e068e4c5a 100644 --- a/src/test/llm/chat_template_end_to_end_minja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_minja_test.cpp @@ -105,7 +105,7 @@ class ChatTemplateEndToEndMinjaTest : public ::testing::Test { } } - // Step 3: Tool probe (only if template supports tools) + // Step 3a: Tool probe (only if template supports tools) if (caps.supportsToolCalls) { ov::genai::Tokenizer probeTokenizer(tokenizerPath); probeTokenizer.set_chat_template(chatTemplate); @@ -115,8 +115,19 @@ class ChatTemplateEndToEndMinjaTest : public ::testing::Test { } } + // Step 3b: Probe reasoning caps using Python Jinja (same function used in production) + { + ov::genai::Tokenizer probeTokenizer(tokenizerPath); + probeTokenizer.set_chat_template(chatTemplate); + bool probeOk = probeChatTemplateReasoning(probeTokenizer, caps); + if (!probeOk) { + std::cout << "=== Reasoning Probe FAILED: minja cannot render reasoning ===" << std::endl; + } + } + std::cout << "=== After Probe ===" << std::endl; std::cout << " requiresObjectArguments: " << caps.requiresObjectArguments << std::endl; + std::cout << " missnamedReasoningField: " << (caps.missnamedReasoningField.empty() ? "(none)" : caps.missnamedReasoningField) << std::endl; // Step 4: Apply workarounds to the chat history chat_template_adapter::applyToHistory(caps, chatHistory); @@ -240,6 +251,7 @@ TEST_F(ChatTemplateEndToEndMinjaTest, Gemma4_ToolCallWithStringArgs) { EXPECT_TRUE(caps.supportsToolCalls); EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); // FIXME: Why is here? because of facebook-opt125? std::string expectedOutput = R"(<|turn>user @@ -276,6 +288,7 @@ TEST_F(ChatTemplateEndToEndMinjaTest, Qwen3Coder_ToolCallWithStringArgs) { EXPECT_FALSE(caps.supportsToolCalls); EXPECT_FALSE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); } // ============================================================================= @@ -301,6 +314,7 @@ TEST_F(ChatTemplateEndToEndMinjaTest, Phi4Mini_ToolCallWithStringArgs) { EXPECT_TRUE(caps.supportsToolCalls); EXPECT_FALSE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); std::string expectedOutput = R"(<|system|> You are a helpful assistant.<|end|><|user|>What's the weather in Paris?<|end|><|assistant|>{"name": "get_weather", "arguments": {"location":"Paris","unit":"celsius"}}<|end|><|assistant|>)"; @@ -330,6 +344,7 @@ TEST_F(ChatTemplateEndToEndMinjaTest, Qwen3_ToolCallWithStringArgs) { EXPECT_TRUE(caps.supportsToolCalls); EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); std::string expectedOutput = R"(<|im_start|>user What's the weather in Paris?<|im_end|> @@ -369,13 +384,14 @@ TEST_F(ChatTemplateEndToEndMinjaTest, Mistral7B_ToolCallWithStringArgs) { EXPECT_TRUE(caps.supportsToolCalls); EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); std::string expectedOutput = R"([INST] What's the weather in Paris?[/INST][TOOL_CALLS] [{"name": "get_weather", "arguments": {"location": "Paris", "unit": "celsius"}, "id": "abc123def"}])"; EXPECT_EQ(appliedOutput, expectedOutput); } // ============================================================================= -// // TODO: Implement assertions, where to take lfm2 deployments steps from? +// // LFM2 does not render tool calls in current chat template, minja inserts them as stringified JSON, which is not correct. // ============================================================================= TEST_F(ChatTemplateEndToEndMinjaTest, LFM2_ToolCallWithStringArgs) { chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_lfm2.jinja"); @@ -388,13 +404,35 @@ TEST_F(ChatTemplateEndToEndMinjaTest, LFM2_ToolCallWithStringArgs) { run(true); - // TODO: Implement assertions, where to take lfm2 deployments steps from? + ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "lfm2"); + ASSERT_FALSE(analysisResult.detectedReasoningParser.has_value()); + + EXPECT_FALSE(caps.supportsToolCalls); + EXPECT_FALSE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); + + std::string expectedOutput = R"(<|im_start|>user +What's the weather in Paris?<|im_end|> +<|im_start|>assistant +{ + "tool_calls": [ + { + "name": "get_weather", + "arguments": { + "location": "Paris", + "unit": "celsius" + }, + "id": "call_abc123" + } + ], + "content": "" +}<|im_end|> +<|im_start|>assistant +)"; + EXPECT_EQ(appliedOutput, expectedOutput); } -// ============================================================================= -// Minja can't handle this chat template for some reason. -// TODO(przepeck): ensure this tests the same template as we will publish to HF -// ============================================================================= TEST_F(ChatTemplateEndToEndMinjaTest, LFM25_ToolCallWithStringArgs) { chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_lfm25.jinja"); ASSERT_FALSE(chatTemplate.empty()) << "Failed to load lfm2.5 template"; @@ -410,14 +448,82 @@ TEST_F(ChatTemplateEndToEndMinjaTest, LFM25_ToolCallWithStringArgs) { ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); EXPECT_EQ(analysisResult.detectedToolParser.value(), "lfm2"); - ASSERT_FALSE(analysisResult.detectedReasoningParser.has_value()); + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "lfm2"); - // TODO: It just does not work for now, documented with assertion + EXPECT_TRUE(caps.supportsToolCalls); + EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_EQ(caps.missnamedReasoningField, "thinking"); - EXPECT_FALSE(caps.supportsToolCalls); - EXPECT_FALSE(caps.requiresObjectArguments); // TODO(przepeck): change once we have it working + std::string expectedOutput = R"(<|im_start|>user +What's the weather in Paris?<|im_end|> +<|im_start|>assistant +<|tool_call_start|>[get_weather(location='Paris', unit='celsius')]<|tool_call_end|><|im_end|> +<|im_start|>assistant +)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + +TEST_F(ChatTemplateEndToEndMinjaTest, LFM25_ToolCallWithStringArgsAndReasoning) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_lfm25.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load lfm2.5 template"; - // TODO: Expect appliedOutput once fixed + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant", "reasoning_content":"Here is some reasoning content","content":"","tool_calls":[{"id":"call_abc123","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"}}]})")); + + run(true); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "lfm2"); + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "lfm2"); + + EXPECT_TRUE(caps.supportsToolCalls); + EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_EQ(caps.missnamedReasoningField, "thinking"); + + std::string expectedOutput = R"(<|im_start|>user +What's the weather in Paris?<|im_end|> +<|im_start|>assistant +Here is some reasoning content<|tool_call_start|>[get_weather(location='Paris', unit='celsius')]<|tool_call_end|><|im_end|> +<|im_start|>assistant +)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + +TEST_F(ChatTemplateEndToEndMinjaTest, LFM25_ReasoningAndEmptyToolCalls) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_lfm25.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load lfm2.5 template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant", "reasoning_content":"Here is some reasoning content","content":"","tool_calls":[]})")); + + run(true); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "lfm2"); + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "lfm2"); + + EXPECT_TRUE(caps.supportsToolCalls); + EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_EQ(caps.missnamedReasoningField, "thinking"); + + std::string expectedOutput = R"(<|im_start|>user +What's the weather in Paris?<|im_end|> +<|im_start|>assistant +Here is some reasoning content<|im_end|> +<|im_start|>assistant +)"; + EXPECT_EQ(appliedOutput, expectedOutput); } // ============================================================================= @@ -442,6 +548,7 @@ TEST_F(ChatTemplateEndToEndMinjaTest, Qwen3VL_ToolCallWithStringArgs) { EXPECT_TRUE(caps.supportsToolCalls); EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); std::string expectedOutput = R"(<|im_start|>user What's the weather in Paris?<|im_end|> @@ -476,6 +583,7 @@ TEST_F(ChatTemplateEndToEndMinjaTest, Qwen3_30B_ToolCallWithStringArgs) { EXPECT_TRUE(caps.supportsToolCalls); EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); std::string expectedOutput = R"(<|im_start|>user What's the weather in Paris?<|im_end|> @@ -506,4 +614,5 @@ TEST_F(ChatTemplateEndToEndMinjaTest, BrokenTemplate_BasicRenderFails) { // Minja silently fails (without exception), but our basic render check should catch it by parsing results. ASSERT_FALSE(exceptionThrownDuringApplication); EXPECT_FALSE(basicRenderOk); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); } diff --git a/src/test/llm/chat_templates/chat_template_lfm25.jinja b/src/test/llm/chat_templates/chat_template_lfm25.jinja index 8bca4a545e..26acf01dfd 100644 --- a/src/test/llm/chat_templates/chat_template_lfm25.jinja +++ b/src/test/llm/chat_templates/chat_template_lfm25.jinja @@ -1,3 +1,7 @@ +{#- + Modifications: + * Adding support for arguments as string so that minja does not detect the need to apply polyfills. This is dead path for OVMS as it always converts arguments to dict anyway. +#} {{- bos_token -}} {%- set preserve_thinking = preserve_thinking | default(false) -%} @@ -34,11 +38,15 @@ {%- for tool_call in tool_calls -%} {%- set func_name = tool_call["function"]["name"] -%} {%- set func_args = tool_call["function"]["arguments"] -%} - {%- set args_ns = namespace(arg_strings=[]) -%} - {%- for arg_name, arg_value in func_args.items() -%} - {%- set args_ns.arg_strings = args_ns.arg_strings + [arg_name + "=" + format_arg_value(arg_value)] -%} - {%- endfor -%} - {%- set tool_calls_ns.tool_calls = tool_calls_ns.tool_calls + [func_name + "(" + (args_ns.arg_strings | join(", ")) + ")"] -%} + {%- if func_args is string -%} + {%- set tool_calls_ns.tool_calls = tool_calls_ns.tool_calls + [func_name + "(" + func_args + ")"] -%} + {%- else -%} + {%- set args_ns = namespace(arg_strings=[]) -%} + {%- for arg_name, arg_value in func_args.items() -%} + {%- set args_ns.arg_strings = args_ns.arg_strings + [arg_name + "=" + format_arg_value(arg_value)] -%} + {%- endfor -%} + {%- set tool_calls_ns.tool_calls = tool_calls_ns.tool_calls + [func_name + "(" + (args_ns.arg_strings | join(", ")) + ")"] -%} + {%- endif -%} {%- endfor -%} {{- "<|tool_call_start|>[" + (tool_calls_ns.tool_calls | join(", ")) + "]<|tool_call_end|>" -}} {%- endmacro -%} @@ -95,7 +103,7 @@ {{- content -}} {%- endif -%} {%- endif -%} - {%- if message.tool_calls is defined -%} + {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 -%} {{- render_tool_calls(message.tool_calls) -}} {%- endif -%} {%- if _has_cfm -%} diff --git a/src/test/llm/output_parsers/lfm25_output_parser_test.cpp b/src/test/llm/output_parsers/lfm25_output_parser_test.cpp new file mode 100644 index 0000000000..fbb6062cad --- /dev/null +++ b/src/test/llm/output_parsers/lfm25_output_parser_test.cpp @@ -0,0 +1,988 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../../llm/io_processing/base_output_parser.hpp" +#include "../../../llm/io_processing/output_parser.hpp" +#include "../../platform_utils.hpp" + +using namespace ovms; + +#ifdef _WIN32 +const std::string tokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\LiquidAI\\LFM2.5-8B-A1B"; +#else +// Hardcoded for usage in docker container +const std::string tokenizerPath = "/ovms/src/test/llm_testing/LiquidAI/LFM2.5-8B-A1B"; +#endif + +static std::unique_ptr lfm25Tokenizer; +static const ToolsSchemas_t& EMPTY_TOOLS_SCHEMA = {}; // not used in lfm2.5 + +class LFM25OutputParserTest : public ::testing::Test { +protected: + std::unique_ptr outputParserWithRegularToolParsing; + + static void SetUpTestSuite() { + try { + lfm25Tokenizer = std::make_unique(tokenizerPath); + } catch (const std::exception& e) { + FAIL() << "Failed to initialize lfm2.5 tokenizer: " << e.what(); + } catch (...) { + FAIL() << "Failed to initialize lfm2 tokenizer due to unknown error."; + } + } + + static void TearDownTestSuite() { + lfm25Tokenizer.reset(); + } + + void SetUp() override { + // For LFM2 model there is only tool parser available + outputParserWithRegularToolParsing = std::make_unique(*lfm25Tokenizer, "lfm2", "lfm2", EMPTY_TOOLS_SCHEMA); + } + + void assertChunkEqual(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk) { + if (!expectedDelta.has_value() && !doc.has_value()) { + return; + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; + } else { + FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk; + } + } + + std::vector encodeChunk(const std::string& chunk) { + if (chunk == "") { + return {124901}; // token ID + } else if (chunk == "") { + return {124902}; // token ID + } + auto generatedTensor = lfm25Tokenizer->encode(chunk, ov::genai::add_special_tokens(false)).input_ids; + std::vector tokensVec(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + return tokensVec; + } + + void assertStreamingVec(const std::vector>>& chunkToDeltaVec) { + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + auto tokens = encodeChunk(chunk); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, tokens, true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; // Both are nullopt, OK + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings + std::string expected = expectedDelta.value(); + std::string idKey = "\"id\":\""; + auto docIdPos = docStr.find(idKey); + auto expectedIdPos = expected.find(idKey); + if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { + auto docIdStart = docIdPos + idKey.size(); + auto docIdEnd = docStr.find("\"", docIdStart); + auto expectedIdStart = expectedIdPos + idKey.size(); + auto expectedIdEnd = expected.find("\"", expectedIdStart); + ASSERT_NE(docIdEnd, std::string::npos); + ASSERT_NE(expectedIdEnd, std::string::npos); + std::string docId = docStr.substr(docIdStart, docIdEnd - docIdStart); + std::string expectedId = expected.substr(expectedIdStart, expectedIdEnd - expectedIdStart); + EXPECT_EQ(docId.size(), expectedId.size()) << "ID length mismatch for chunk: " << chunk; + EXPECT_TRUE(std::all_of(docId.begin(), docId.end(), ::isalnum)) << "ID not alphanumeric for chunk: " << chunk; + // Compare everything except the id value + std::string docStrNoId = docStr; + std::string expectedNoId = expected; + docStrNoId.replace(docIdStart, docId.size(), std::string(docId.size(), '*')); + expectedNoId.replace(expectedIdStart, expectedId.size(), std::string(expectedId.size(), '*')); + EXPECT_EQ(docStrNoId, expectedNoId) << "Mismatch for chunk (ignoring id value): " << chunk; + } else { + EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; + } + } else { + std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; + std::string docStr = doc.has_value() ? [&]() { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + return std::string(buffer.GetString()); + }() + : "std::nullopt"; + FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk + << "\nexpectedDelta: " << expectedStr + << "\ndoc: " << docStr; + } + } + } +}; + +TEST_F(LFM25OutputParserTest, ParseToolCallOutputWithSingleToolCall) { + std::string inputWithProperClosure = "<|tool_call_start|>[example_tool(arg1=\"value1\", arg2=42)]<|tool_call_end|>"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + } +} + +TEST_F(LFM25OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { + std::string inputWithProperClosure = "<|tool_call_start|>[example_tool(arg1=\"value1\", arg2=42)]<|tool_call_end|>"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + std::string testInput = input; + auto generatedTensor = lfm25Tokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, false); + EXPECT_EQ(parsedOutput.content, testInput); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 0); + } +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithObjectArguments) { + std::string inputWithProperClosure = "<|tool_call_start|>[dummy(config={'name': 'astro_config', 'value': 99})]<|tool_call_end|>"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "dummy"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"config\":{\"name\":\"astro_config\",\"value\":99}}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + } +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArguments) { + std::string inputWithProperClosure = "<|tool_call_start|>[test1(arg1=\"data1, data2\")]<|tool_call_end|>"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "test1"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"data1, data2\"}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + } +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithListOfStringsAsArgument) { + std::string inputWithProperClosure = "<|tool_call_start|>[generate_DNA_sequence(length=100, preferences=['G', 'C'])]<|tool_call_end|>"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "generate_DNA_sequence"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"length\":100,\"preferences\":[\"G\",\"C\"]}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + } +} + +TEST_F(LFM25OutputParserTest, ParserToolCallWithBooleanArgument) { + std::string inputWithProperClosure = "<|tool_call_start|>[check_status(flag=True)]<|tool_call_end|>"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "check_status"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"flag\":true}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + } +} + +TEST_F(LFM25OutputParserTest, ParseTwoToolCallsAtOnce) { + std::string inputWithProperClosure = "<|tool_call_start|>[dummy1(config={'name': 'astro_config', 'value': 99}), dummy2(config={'name': 'second_config', 'value': 199})]<|tool_call_end|>"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 2); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "dummy1"); + EXPECT_EQ(parsedOutput.toolCalls[1].name, "dummy2"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"config\":{\"name\":\"astro_config\",\"value\":99}}"); + EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"config\":{\"name\":\"second_config\",\"value\":199}}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + EXPECT_EQ(parsedOutput.toolCalls[1].id.empty(), false); // ID should be generated + } +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithArrayArguments) { + std::string inputWithProperClosure = "<|tool_call_start|>[sort(array=[42, 17, 89, 5, 33], order=\"descending\")]<|tool_call_end|>"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "sort"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + } +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithStringWithSingleQuotesArguments) { + std::string inputWithProperClosure = "<|tool_call_start|>[sort(array=[42, 17, 89, 5, 33], order='descending')]<|tool_call_end|>"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "sort"); + // Parser removes whitespaces, so we expect arguments value to be without spaces + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated + } +} + +TEST_F(LFM25OutputParserTest, ParseToolCallOutputWithThreeToolCalls) { + std::string inputWithProperClosure = "<|tool_call_start|>[example_tool(arg1=\"value1\", arg2=42)]<|tool_call_end|>" + "<|tool_call_start|>[another_tool(param1=\"data\", param2=true)]<|tool_call_end|>" + "<|tool_call_start|>[third_tool(key=\"value\")]<|tool_call_end|>"; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 3); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); + auto firstToolCallId = parsedOutput.toolCalls[0].id; + + EXPECT_EQ(parsedOutput.toolCalls[1].name, "another_tool"); + EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"param1\":\"data\",\"param2\":true}"); + EXPECT_EQ(parsedOutput.toolCalls[1].id.empty(), false); + auto secondToolCallId = parsedOutput.toolCalls[1].id; + EXPECT_NE(firstToolCallId, secondToolCallId); + + EXPECT_EQ(parsedOutput.toolCalls[2].name, "third_tool"); + EXPECT_EQ(parsedOutput.toolCalls[2].arguments, "{\"key\":\"value\"}"); + EXPECT_EQ(parsedOutput.toolCalls[2].id.empty(), false); + auto thirdToolCallId = parsedOutput.toolCalls[2].id; + EXPECT_NE(firstToolCallId, thirdToolCallId); + EXPECT_NE(secondToolCallId, thirdToolCallId); + } +} + +TEST_F(LFM25OutputParserTest, ParseToolCallOutputWithThreeToolCallsWithContentInBetween) { + std::string inputWithProperClosure = "Before tool calls content. " + "<|tool_call_start|>[example_tool(arg1=\"value1\", arg2=42)]<|tool_call_end|>" + "This is some content between tool calls." + "<|tool_call_start|>[another_tool(param1=\"data\", param2=true)]<|tool_call_end|>" + " This is some content between second and third tool call. " + "<|tool_call_start|>[third_tool(key=\"value\")]<|tool_call_end|>" + "After tool calls content."; + + std::vector inputs = {inputWithProperClosure}; + for (auto& input : inputs) { + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, "Before tool calls content. This is some content between tool calls. This is some content between second and third tool call. After tool calls content."); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 3); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); + auto firstToolCallId = parsedOutput.toolCalls[0].id; + + EXPECT_EQ(parsedOutput.toolCalls[1].name, "another_tool"); + EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"param1\":\"data\",\"param2\":true}"); + EXPECT_EQ(parsedOutput.toolCalls[1].id.empty(), false); + auto secondToolCallId = parsedOutput.toolCalls[1].id; + EXPECT_NE(firstToolCallId, secondToolCallId); + + EXPECT_EQ(parsedOutput.toolCalls[2].name, "third_tool"); + EXPECT_EQ(parsedOutput.toolCalls[2].arguments, "{\"key\":\"value\"}"); + EXPECT_EQ(parsedOutput.toolCalls[2].id.empty(), false); + auto thirdToolCallId = parsedOutput.toolCalls[2].id; + EXPECT_NE(firstToolCallId, thirdToolCallId); + EXPECT_NE(secondToolCallId, thirdToolCallId); + } +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithEmptyArguments) { + // Tool call with empty parentheses (no arguments) + std::string input = "<|tool_call_start|>[no_args_tool()]<|tool_call_end|>"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "no_args_tool"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallOutputWithContentAndNoToolCalls) { + std::string input = "This is a regular model response without tool calls."; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, "This is a regular model response without tool calls."); + ASSERT_EQ(parsedOutput.toolCalls.size(), 0); + EXPECT_EQ(parsedOutput.reasoning, ""); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallOutputWithContentAndSingleToolCall) { + std::string input = "This is a content part and next will be a tool call.\n\n<|tool_call_start|>[example_tool(arg1=\"value1\", arg2=42)]<|tool_call_end|>"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, "This is a content part and next will be a tool call.\n\n"); + EXPECT_EQ(parsedOutput.reasoning, ""); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); +} + +TEST_F(LFM25OutputParserTest, HolisticStreaming) { + std::vector>> chunkToDeltaVec{ + {"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG"}})"}, + {"<|tool_call_start|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"sort", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"(array", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"sort"}}]}})"}, + {"=[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"42", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 17", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 89", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 5", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 33", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"],", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" order", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"=\"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"desc", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ending", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\"),", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, + {" d", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ummy", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"(config", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":1,"function":{"name":"dummy"}}]}})"}, + {"={", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"'", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"name", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"astro_config", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"',", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"value", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 99", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"})]", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"config\":{\"name\":\"astro_config\",\"value\":99}}"}}]}})"}, + {"<|tool_call_end|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ANOTHER_CONTENT_AFTER_TOOL_CALL", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"ANOTHER_CONTENT_AFTER_TOOL_CALL"}})"}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST_F(LFM25OutputParserTest, StreamingWithBiggerChunks) { + std::vector>> chunkToDeltaVec{ + {"SOME_CONTENT", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"SOME_CONTENT"}})"}, + {"MORE_CONTENT<|tool_call_start|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"MORE_CONTENT"}})"}, + {"[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"sort(array=", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"sort"}}]}})"}, + {"[42, 17, 89, 5, 33], order=\"descending\"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {")]", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, + {"<|tool_call_end|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ANOTHER_CONTENT_AFTER_TOOL_CALL", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"ANOTHER_CONTENT_AFTER_TOOL_CALL"}})"}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST_F(LFM25OutputParserTest, StreamingWithToolCallAndFinishReason) { + std::vector>> chunkToDeltaVec{ + {"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG"}})"}, + {"<|tool_call_start|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" d", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ummy", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"(config", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"dummy"}}]}})"}, + {"={", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"'", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"name", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"astro_config", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"',", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"value", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 99", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"})]", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"config\":{\"name\":\"astro_config\",\"value\":99}}"}}]}})"}, + {"<|tool_call_end|><|im_end|>", ov::genai::GenerationFinishReason::STOP, std::nullopt}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST_F(LFM25OutputParserTest, StreamingWithToolCallAndEOSToken) { + std::vector>> chunkToDeltaVec{ + {"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG"}})"}, + {"<|tool_call_start|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" d", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ummy", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"(config", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"dummy"}}]}})"}, + {"={", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"'", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"name", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"astro_config", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"',", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"value", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 99", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"})]", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"config\":{\"name\":\"astro_config\",\"value\":99}}"}}]}})"}, + {"<|tool_call_end|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|im_end|>", ov::genai::GenerationFinishReason::STOP, std::nullopt}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST_F(LFM25OutputParserTest, StreamingWithToolCallAndEOSTokenAndContent) { + std::vector>> chunkToDeltaVec{ + {"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG"}})"}, + {"<|tool_call_start|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" d", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ummy", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"(config", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"dummy"}}]}})"}, + {"={", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"'", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"name", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"astro_config", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"',", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"value", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 99", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"})]", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"config\":{\"name\":\"astro_config\",\"value\":99}}"}}]}})"}, + {"<|tool_call_end|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"SOME_CONTENT<|im_end|>", ov::genai::GenerationFinishReason::STOP, R"({"delta":{"content":"SOME_CONTENT"}})"}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST_F(LFM25OutputParserTest, StreamingWithContentBetweenToolCalls) { + std::vector>> chunkToDeltaVec{ + // Tool call phase + // Starting first tool. Collecting chunk until full name is received. Don't return until then. + {"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG"}})"}, + {"<|tool_call_start|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"sort", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"(array", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"sort"}}]}})"}, + {"=[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"42", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 17", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 89", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 5", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 33", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"],", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" order", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"=\"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"desc", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ending", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\")]", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, + {"<|tool_call_end|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"Some ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"Some "}})"}, + {"content ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"content "}})"}, + {"between ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"between "}})"}, + {"tool ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"tool "}})"}, + {"calls.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"calls."}})"}, + {"<|tool_call_start|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"d", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ummy", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"(config", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":1,"function":{"name":"dummy"}}]}})"}, + {"={", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"'", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"name", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"astro_config", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"',", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" '", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"value", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"':", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 99", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"})]", ov ::genai ::GenerationFinishReason ::NONE, R"({"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"config\":{\"name\":\"astro_config\",\"value\":99}}"}}]}})"}, + {"<|tool_call_end|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ANOTHER_CONTENT_AFTER_TOOL_CALL", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"ANOTHER_CONTENT_AFTER_TOOL_CALL"}})"}, + {"<|tool_call_start|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"[solve", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"(e", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":2,"function":{"name":"solve"}}]}})"}, + {"quation", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"=\"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"2", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"*", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"(", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"x", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"+", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"5)", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" =", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 13", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\")]", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":2,"function":{"arguments":"{\"equation\":\"2*(x+5) = 13\"}"}}]}})"}, + {"<|tool_call_end|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"And some content after second tool call", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"And some content after second tool call"}})"}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST_F(LFM25OutputParserTest, StreamingWithReasoningAndToolCall) { + std::vector>> chunkToDeltaVec{ + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"I ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"I "}})"}, + {"need ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"need "}})"}, + {"to ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"to "}})"}, + {"sort ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"sort "}})"}, + {"an ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"an "}})"}, + {"array ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"array "}})"}, + {"of ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"of "}})"}, + {"numbers ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"numbers "}})"}, + {"in ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"in "}})"}, + {"descending ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"descending "}})"}, + {"order.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"order."}})"}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|tool_call_start|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"sort", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"(array", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"sort"}}]}})"}, + {"=[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"42", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 17", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 89", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 5", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 33", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"],", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" order", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"=\"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"desc", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ending", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\")]", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, + {"<|tool_call_end|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST_F(LFM25OutputParserTest, StreamingWithReasoningAndToolCallAndContent) { + std::vector>> chunkToDeltaVec{ + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"I ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"I "}})"}, + {"need ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"need "}})"}, + {"to ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"to "}})"}, + {"sort ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"sort "}})"}, + {"an ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"an "}})"}, + {"array ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"array "}})"}, + {"of ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"of "}})"}, + {"numbers ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"numbers "}})"}, + {"in ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"in "}})"}, + {"descending ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"descending "}})"}, + {"order.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"order."}})"}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|tool_call_start|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"sort", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"(array", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"sort"}}]}})"}, + {"=[", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"42", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 17", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 89", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 5", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {",", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" 33", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"],", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" order", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"=\"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"desc", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ending", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\")]", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"}}]}})"}, + {"<|tool_call_end|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"Some", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"Some"}})"}, + {" content", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" content"}})"}, + {" after", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" after"}})"}, + {" the", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" the"}})"}, + {" tool", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" tool"}})"}, + {" call.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" call."}})"}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST_F(LFM25OutputParserTest, ToolCallsWithoutToolsInTheRequestStreaming) { + std::vector>> chunkToDeltaVec{ + // Tool parser is available, but tools are not in the request so every chunk is just a regular content + {"<|tool_call_start|>\n", "{\"delta\":{\"content\":\"<|tool_call_start|>\\n\"}}"}, + {"{\"", "{\"delta\":{\"content\":\"{\\\"\"}}"}, + {"name", "{\"delta\":{\"content\":\"name\"}}"}, + {"\":", "{\"delta\":{\"content\":\"\\\":\"}}"}, + {" \"", "{\"delta\":{\"content\":\" \\\"\"}}"}, + {"super", "{\"delta\":{\"content\":\"super\"}}"}, + {"_tool", "{\"delta\":{\"content\":\"_tool\"}}"}, + {"_number", "{\"delta\":{\"content\":\"_number\"}}"}, + {"_two", "{\"delta\":{\"content\":\"_two\"}}"}, + {"\",", "{\"delta\":{\"content\":\"\\\",\"}}"}, + {" \"", "{\"delta\":{\"content\":\" \\\"\"}}"}, + {"arguments", "{\"delta\":{\"content\":\"arguments\"}}"}, + {"\":", "{\"delta\":{\"content\":\"\\\":\"}}"}, + {" {", "{\"delta\":{\"content\":\" {\"}}"}, + {"\"", "{\"delta\":{\"content\":\"\\\"\"}}"}, + {"arg1", "{\"delta\":{\"content\":\"arg1\"}}"}, + {"\": ", "{\"delta\":{\"content\":\"\\\": \"}}"}, + {"\"", "{\"delta\":{\"content\":\"\\\"\"}}"}, + {"val{{{ue1", "{\"delta\":{\"content\":\"val{{{ue1\"}}"}, + {"\"", "{\"delta\":{\"content\":\"\\\"\"}}"}, + {"}", "{\"delta\":{\"content\":\"}\"}}"}, + {"}", "{\"delta\":{\"content\":\"}\"}}"}, + {"<|tool_call_end|>\n", "{\"delta\":{\"content\":\"<|tool_call_end|>\\n\"}}"}, + }; + + for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { + // Second argument is false as we simulate the case where tools have not been provided in the request + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); + assertChunkEqual(doc, expectedDelta, chunk); + } +} + +// Malformed tool calls + +TEST_F(LFM25OutputParserTest, ParseToolCallWithMissingParentheses) { + std::string input = "<|tool_call_start|>[broken_tool]<|tool_call_end|>"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ASSERT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithMissingClosingParenthesis) { + std::string input = "<|tool_call_start|>[broken_tool(arg1=\"value1\"]<|tool_call_end|>"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ASSERT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithArgumentMissingEquals) { + // Argument without '=' sign - parseSingleArgument sets isValid = false + std::string input = "<|tool_call_start|>[broken(malformed_arg)]<|tool_call_end|>"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + // The tool call is parsed but the argument value will be empty and invalid + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "broken"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithMissingSquareBracket) { + std::string input = "<|tool_call_start|>broken(arg1=1)<|tool_call_end|>"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ASSERT_EQ(parsedOutput.toolCalls.size(), 0); +} + +// Tests with special characters + +TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsContainingComparison) { + std::string input = R"x(<|tool_call_start|>[search(query="price >= 100, (sale)", limit=5)]<|tool_call_end|>)x"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "search"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"x({"query":"price >= 100, (sale)","limit":5})x"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsContainingBracesAndBrackets) { + std::string input = R"(<|tool_call_start|>[format(template="Hello {name}, items: [a, b, c]", count=3)]<|tool_call_end|>)"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "format"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"template":"Hello {name}, items: [a, b, c]","count":3})"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsContainingSpecialCharacters) { + std::string impl = "import package\nimport package2\n\ndef func(a, b):\n\td={\"python\": \"dict\"}\n\tl = [\"list \\\"with escaped text\\\"\", 123, []]\n\treturn f\"formatted {a} and {b}\""; + std::string input = R"(<|tool_call_start|>[execute(code=")" + impl + R"(")]<|tool_call_end|>)"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "execute"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"code":"import package\nimport package2\n\ndef func(a, b):\n\td={\"python\": \"dict\"}\n\tl = [\"list \\\"with escaped text\\\"\", 123, []]\n\treturn f\"formatted {a} and {b}\""})"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsContainingEscapedQuotes) { + std::string input = R"x(<|tool_call_start|>[execute(code="print(\"hello world\")", verbose=true)]<|tool_call_end|>)x"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "execute"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"x({"code":"print(\"hello world\")","verbose":true})x"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsContainingApostrophes) { + std::string input = R"(<|tool_call_start|>[log(message="it's a test, isn't it?", level="warn")]<|tool_call_end|>)"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "log"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"message":"it's a test, isn't it?","level":"warn"})"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsContainingBackslashes) { + std::string input = R"(<|tool_call_start|>[read_file(path="C:\Users\test\file.txt", encoding="utf-8")]<|tool_call_end|>)"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "read_file"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"path":"C:\\Users\\test\\file.txt","encoding":"utf-8"})"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsArrayWithStringsContainingQuotes) { + std::string input = R"(<|tool_call_start|>[save(lines=['it's the wonderful day', 'My name's Jan', 'That's Johns' car.'])]<|tool_call_end|>)"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "save"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"lines":["it's the wonderful day","My name's Jan","That's Johns' car."]})"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentContainingSingleQuotes) { + std::string input = R"(<|tool_call_start|>[save(line="I've had line with single quotes")]<|tool_call_end|>)"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "save"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"line":"I've had line with single quotes"})"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsObjectWithStringsContainingQuotes) { + std::string input = R"(<|tool_call_start|>[save(obj={'name':'it's the wonderful day', 'greeting':'Hello, my name's Jan', 'note':'That's Johns' car.'})]<|tool_call_end|>)"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "save"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"obj":{"name":"it's the wonderful day","greeting":"Hello, my name's Jan","note":"That's Johns' car."}})"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsContainingNestedJSON) { + std::string input = R"(<|tool_call_start|>[send(payload="{'key': 'value', 'count': 42}", endpoint="api")]<|tool_call_end|>)"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "send"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"payload":"{'key': 'value', 'count': 42}","endpoint":"api"})"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithEmptyStringArgument) { + std::string input = R"(<|tool_call_start|>[create(name="", value=0)]<|tool_call_end|>)"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "create"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"name":"","value":0})"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithUnicodeCharactersInArguments) { + std::string input = R"(<|tool_call_start|>[translate(text="zażółć gęślą jaźń", lang="pl")]<|tool_call_end|>)"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "translate"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"text":"zażółć gęślą jaźń","lang":"pl"})"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithPythonCodeAsArgument) { + std::string input = R"x(<|tool_call_start|>[string_tool(param=" + if __name__ == "__main__": + addresses = {} + addresses["Hodor"] = """The door""" + addresses["Arya"] = "Winterfell" + for name, address in addresses.items(): + print(f'\n\t{name} lives at {address}\n\r')")]<|tool_call_end|>)x"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "string_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"x({"param":"\n if __name__ == \"__main__\":\n addresses = {}\n addresses[\"Hodor\"] = \"\"\"The door\"\"\"\n addresses[\"Arya\"] = \"Winterfell\"\n for name, address in addresses.items():\n print(f'\\n\\t{name} lives at {address}\\n\\r')"})x"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithReasoning) { + std::string input = R"(User wants me to translate string "zażółć gęślą jaźń" from polish. Polish parameter language signature is "pl". I should use function translate. [...]<|tool_call_start|>[translate(text="zażółć gęślą jaźń", lang="pl")]<|tool_call_end|>)"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "translate"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"text":"zażółć gęślą jaźń","lang":"pl"})"); + EXPECT_EQ(parsedOutput.reasoning, R"(User wants me to translate string "zażółć gęślą jaźń" from polish. Polish parameter language signature is "pl". I should use function translate. [...])"); +} + +TEST_F(LFM25OutputParserTest, ParseToolCallWithReasoningAndContent) { + std::string input = R"(User wants me to translate string "zażółć gęślą jaźń" from polish. Polish parameter language signature is "pl". I should use function translate. [...]<|tool_call_start|>[translate(text="zażółć gęślą jaźń", lang="pl")]<|tool_call_end|> This is the content after the tool call.)"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, " This is the content after the tool call."); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "translate"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"text":"zażółć gęślą jaźń","lang":"pl"})"); + EXPECT_EQ(parsedOutput.reasoning, R"(User wants me to translate string "zażółć gęślą jaźń" from polish. Polish parameter language signature is "pl". I should use function translate. [...])"); +} + +TEST_F(LFM25OutputParserTest, ParseOutputWithReasoningAndContent) { + std::string input = R"(User wants me to answer what is the difference between "foo" and "bar". I should answer with a short explanation. [...] The difference between "foo" and "bar" is that "foo" is often used as a placeholder name in programming, while "bar" is another placeholder name that is commonly used alongside "foo".)"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, " The difference between \"foo\" and \"bar\" is that \"foo\" is often used as a placeholder name in programming, while \"bar\" is another placeholder name that is commonly used alongside \"foo\"."); + ASSERT_EQ(parsedOutput.toolCalls.size(), 0); + EXPECT_EQ(parsedOutput.reasoning, R"(User wants me to answer what is the difference between "foo" and "bar". I should answer with a short explanation. [...])"); +} + +TEST_F(LFM25OutputParserTest, ParseOutputWithoutReasoningAndTools) { + std::string input = R"(This is a simple output without reasoning and tools.)"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + EXPECT_EQ(parsedOutput.content, "This is a simple output without reasoning and tools."); + ASSERT_EQ(parsedOutput.toolCalls.size(), 0); + EXPECT_EQ(parsedOutput.reasoning, ""); +} diff --git a/src/test/llm/output_parsers/lfm2_output_parser_test.cpp b/src/test/llm/output_parsers/lfm2_output_parser_test.cpp index e20920aa6b..ff5bdf62b7 100644 --- a/src/test/llm/output_parsers/lfm2_output_parser_test.cpp +++ b/src/test/llm/output_parsers/lfm2_output_parser_test.cpp @@ -167,8 +167,6 @@ TEST_F(LFM2OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { TEST_F(LFM2OutputParserTest, ParseToolCallWithObjectArguments) { std::string inputWithProperClosure = "<|tool_call_start|>[dummy(config={'name': 'astro_config', 'value': 99})]<|tool_call_end|>"; - // LFM2 may produce last tool call without closing tag, so we test both cases - // The results should be identical std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; @@ -188,8 +186,6 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithObjectArguments) { TEST_F(LFM2OutputParserTest, ParseToolCallWithStringArguments) { std::string inputWithProperClosure = "<|tool_call_start|>[test1(arg1=\"data1, data2\")]<|tool_call_end|>"; - // LFM2 may produce last tool call without closing tag, so we test both cases - // The results should be identical std::vector inputs = {inputWithProperClosure}; for (auto& input : inputs) { auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; @@ -601,7 +597,7 @@ TEST_F(LFM2OutputParserTest, StreamingWithContentBetweenToolCalls) { {"<|tool_call_end|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"ANOTHER_CONTENT_AFTER_TOOL_CALL", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"ANOTHER_CONTENT_AFTER_TOOL_CALL"}})"}, {"<|tool_call_start|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"solve", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"[solve", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"(e", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":2,"function":{"name":"solve"}}]}})"}, {"quation", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"=\"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, @@ -685,7 +681,27 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithArgumentMissingEquals) { EXPECT_EQ(parsedOutput.toolCalls[0].name, "broken"); } +TEST_F(LFM2OutputParserTest, ParseToolCallWithArgumentMissingValue) { + // Argument without value - parseSingleArgument sets isValid = false + std::string input = "<|tool_call_start|>[broken(arg1=)]<|tool_call_end|>"; + auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + // The tool call is parsed but the argument value will be empty and invalid + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "broken"); +} + +TEST_F(LFM2OutputParserTest, ParseToolCallWithMissingSquareBracket) { + std::string input = "<|tool_call_start|>broken(arg1=1)<|tool_call_end|>"; + auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ASSERT_EQ(parsedOutput.toolCalls.size(), 0); +} + // Tests with special characters + TEST_F(LFM2OutputParserTest, ParseToolCallWithStringArgumentsContainingComparison) { std::string input = R"x(<|tool_call_start|>[search(query="price >= 100, (sale)", limit=5)]<|tool_call_end|>)x"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; diff --git a/src/test/llm/output_parsers/parser_config_validation_test.cpp b/src/test/llm/output_parsers/parser_config_validation_test.cpp index 2250675d45..367a4e32a9 100644 --- a/src/test/llm/output_parsers/parser_config_validation_test.cpp +++ b/src/test/llm/output_parsers/parser_config_validation_test.cpp @@ -73,7 +73,7 @@ TEST_F(ParserConfigValidationTest, NoneIsAcceptedAsDisabledParser) { TEST_F(ParserConfigValidationTest, RegistryHasExpectedReasoningParsers) { const auto& names = getSupportedReasoningParserNames(); - for (const auto& expected : {"qwen3", "gemma4", "gptoss"}) { + for (const auto& expected : {"qwen3", "gemma4", "gptoss", "lfm2"}) { EXPECT_NE(std::find(names.begin(), names.end(), expected), names.end()) << "Expected reasoning parser '" << expected << "' missing from registry"; } @@ -89,6 +89,7 @@ TEST_F(ParserConfigValidationTest, SupportedNamesStringContainsAllParsers) { const std::string reasoningNames = getSupportedReasoningParserNamesAsString(); EXPECT_NE(reasoningNames.find("qwen3"), std::string::npos); EXPECT_NE(reasoningNames.find("gptoss"), std::string::npos); + EXPECT_NE(reasoningNames.find("lfm2"), std::string::npos); } TEST_F(ParserConfigValidationTest, OutputParserThrowsOnUnknownToolParser) { diff --git a/windows_prepare_llm_models.bat b/windows_prepare_llm_models.bat index c49fdc1f0f..48f13e61b8 100644 --- a/windows_prepare_llm_models.bat +++ b/windows_prepare_llm_models.bat @@ -45,6 +45,7 @@ set "MISTRAL_MODEL=mistralai/Mistral-7B-Instruct-v0.3" set "GPTOSS_MODEL=openai/gpt-oss-20b" set "DEVSTRAL_MODEL=unsloth/Devstral-Small-2507" set "LFM2_MODEL=LiquidAI/LFM2-2.6B" +set "LFM25_MODEL=LiquidAI/LFM2.5-8B-A1B" set "GEMMA4_MODEL=OpenVINO/gemma-4-E4B-it-int4-ov" echo Downloading LLM testing models to directory %~1 @@ -86,6 +87,7 @@ call :download_tokenizer "%MISTRAL_MODEL%" "%~1\%MISTRAL_MODEL%" call :download_tokenizer "%GPTOSS_MODEL%" "%~1\%GPTOSS_MODEL%" call :download_tokenizer "%DEVSTRAL_MODEL%" "%~1\%DEVSTRAL_MODEL%" call :download_tokenizer "%LFM2_MODEL%" "%~1\%LFM2_MODEL%" +call :download_tokenizer "%LFM25_MODEL%" "%~1\%LFM25_MODEL%" call :download_openvino_tokenizer "%GEMMA4_MODEL%" "%~1" exit /b 0