From 220c122c8ad098c2f6dabf3484826467961ffded Mon Sep 17 00:00:00 2001 From: divyanarahari97 Date: Mon, 10 Aug 2026 12:24:52 -0700 Subject: [PATCH] fix(sampling): keep stop sequence token ids aligned with their strings stop_sentences_to_token_ids() drops entries that encode to no tokens (e.g. "", [], or any string the tokenizer maps to nothing), but StopSequenceGroups.initialize() then indexed the *original* stop_sequences list by the *filtered* list's index. Every entry after a dropped one shifted by one position, so a group's sequence_str came from the wrong stop entry. Two user-visible failures: stop=["", "END"] -> to_strings() == [] "END" silently loses string matching entirely. stop=["unknown", "stop2"] -> to_strings() == ["unknown"] generation stops on a string the user never requested, because "unknown"'s string got attached to "stop2"'s token ids. The second is the damaging one: these strings drive stop matching in DecodeReq.stop_sequences_str_match() and trailing-stop trimming in the OpenAI completion path, so a request can terminate early on unrelated text. Carry (token_ids, original_entry) pairs through the filter so the two can never drift apart. stop_sentences_to_token_ids() keeps its original signature as a thin wrapper over the new helper. Also repairs the test module, which imported a DecodeNode class that no longer exists in sampling_params and therefore failed at collection -- meaning none of these tests had been running. Replaced with an equivalent NodeUUId round-trip test and added regression coverage for the alignment bug above. Co-Authored-By: Claude Opus 5 --- lightllm/server/core/objs/sampling_params.py | 27 +++++++---- .../server/core/objs/test_sampling_params.py | 47 ++++++++++++------- 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/lightllm/server/core/objs/sampling_params.py b/lightllm/server/core/objs/sampling_params.py index 8e31c50624..623bd52d30 100644 --- a/lightllm/server/core/objs/sampling_params.py +++ b/lightllm/server/core/objs/sampling_params.py @@ -66,31 +66,42 @@ def initialize(self, stop_sequences: Union[str, List[Union[List[int], str]]], to elif isinstance(stop_sequences, str): stop_sequences = [stop_sequences] - groups: List[List[int]] = self.stop_sentences_to_token_ids(stop_sequences, tokenizer) + # 这里必须使用 (token_ids, 原始条目) 的配对结果：空条目（如 "" 或 []）会被过滤掉， + # 若还按原始下标去取 stop_sequences[group_idx]，后面的条目就会和 token id 组错位， + # 导致 sequence_str 挂到别的组上（丢失停止字符串，或凭空多出一个停止字符串）。 + groups: List[Tuple[List[int], Union[List[int], str]]] = self._stop_sentences_to_token_id_pairs( + stop_sequences, tokenizer + ) self.size = len(groups) assert self.size <= MAX_STOP_SEQUENCES, "Too many stop sequence groups." - for group_idx in range(self.size): - if isinstance(stop_sequences[group_idx], str): - self.groups[group_idx].initialize(groups[group_idx], sequence_str=stop_sequences[group_idx]) + for group_idx, (token_ids, stop_info) in enumerate(groups): + if isinstance(stop_info, str): + self.groups[group_idx].initialize(token_ids, sequence_str=stop_info) else: - self.groups[group_idx].initialize(groups[group_idx]) + self.groups[group_idx].initialize(token_ids) - def stop_sentences_to_token_ids(self, stop_sequences: List[Union[List[int], str]], tokenizer) -> List[List[int]]: + def _stop_sentences_to_token_id_pairs( + self, stop_sequences: List[Union[List[int], str]], tokenizer + ) -> List[Tuple[List[int], Union[List[int], str]]]: + """返回 (token_ids, 原始 stop 条目) 的列表，保证两者一一对应。""" new_stop_sequences = [] for stop_info in stop_sequences: if isinstance(stop_info, str): stop_str_ids = self._stop_str_to_token_ids(stop_info, tokenizer) if stop_str_ids is not None and len(stop_str_ids) > 0: - new_stop_sequences.append(stop_str_ids) + new_stop_sequences.append((stop_str_ids, stop_info)) if isinstance(stop_info, list): if all(isinstance(x, int) for x in stop_info): if len(stop_info) > 0: - new_stop_sequences.append(stop_info) + new_stop_sequences.append((stop_info, stop_info)) else: assert False, "stop_sequences item must be type List[int] when it is a list." return new_stop_sequences + def stop_sentences_to_token_ids(self, stop_sequences: List[Union[List[int], str]], tokenizer) -> List[List[int]]: + return [token_ids for token_ids, _ in self._stop_sentences_to_token_id_pairs(stop_sequences, tokenizer)] + def _stop_str_to_token_ids(self, stop_str: str, tokenizer) -> List[int]: stop_str_ids = tokenizer.encode(stop_str, add_special_tokens=False) return stop_str_ids diff --git a/unit_tests/server/core/objs/test_sampling_params.py b/unit_tests/server/core/objs/test_sampling_params.py index ef3f08d2fc..a1c59e1564 100644 --- a/unit_tests/server/core/objs/test_sampling_params.py +++ b/unit_tests/server/core/objs/test_sampling_params.py @@ -5,10 +5,10 @@ RegularConstraint, AllowedTokenIds, ExponentialDecayLengthPenalty, - DecodeNode, SamplingParams, GuidedGrammar, GuidedJsonSchema, + NodeUUId, STOP_SEQUENCE_MAX_LENGTH, REGULAR_CONSTRAINT_MAX_LENGTH, ALLOWED_TOKEN_IDS_MAX_LENGTH, @@ -117,22 +117,35 @@ def test_exponential_decay_length_penalty_initialization(): penalty.initialize((5, 0.5)) -def test_decode_node_initialization(): - node = DecodeNode() - data = { - "node_id": 12345678901234567890, # 示例 UUID - "ip": "192.168.1.1", - "rpyc_port": 8080, - "max_new_tokens": 10, - } - node.initialize(data) - assert node.exists is True - assert node.node_id.node_id_high == (12345678901234567890 >> 64) & 0xFFFFFFFFFFFFFFFF - assert node.node_id.node_id_low == 12345678901234567890 & 0xFFFFFFFFFFFFFFFF - assert node.ip[0] == 192 - assert node.ip[1] == 168 - assert node.ip[2] == 1 - assert node.ip[3] == 1 +def test_node_uuid_roundtrip(): + node_id = 12345678901234567890 + uuid = NodeUUId() + uuid.initialize(node_id) + assert uuid.node_id_high == (node_id >> 64) & 0xFFFFFFFFFFFFFFFF + assert uuid.node_id_low == node_id & 0xFFFFFFFFFFFFFFFF + assert uuid.get() == node_id + + +@pytest.mark.parametrize( + "stop_sequences, expected_token_ids, expected_strings", + [ + # 全部有效，无过滤，token id 组与字符串一一对应。 + (["stop1", "stop2"], [[1, 2], [3, 4]], ["stop1", "stop2"]), + # 前置的空字符串会被过滤掉，后面的条目不能因此错位。 + (["", "stop1"], [[1, 2]], ["stop1"]), + # 前置的空 id 列表同理。 + ([[], "stop2"], [[3, 4]], ["stop2"]), + # 被过滤掉的字符串条目不能把自己的字符串挂到后一个条目的 token id 上。 + (["unknown", "stop2"], [[3, 4]], ["stop2"]), + # 纯 id 条目不携带字符串。 + ([[7, 8], "stop1"], [[7, 8], [1, 2]], ["stop1"]), + ], +) +def test_stop_sequence_groups_keeps_ids_and_strings_aligned(stop_sequences, expected_token_ids, expected_strings): + groups = StopSequenceGroups() + groups.initialize(stop_sequences, MockTokenizer()) + assert groups.to_list() == expected_token_ids + assert sorted(groups.to_strings()) == sorted(expected_strings) def test_sampling_params_initialization():