From 220c122c8ad098c2f6dabf3484826467961ffded Mon Sep 17 00:00:00 2001 From: divyanarahari97 Date: Mon, 10 Aug 2026 12:24:52 -0700 Subject: [PATCH 1/2] fix(sampling): keep stop sequence token ids aligned with their strings stop_sentences_to_token_ids() drops entries that encode to no tokens (e.g. "", [], or any string the tokenizer maps to nothing), but StopSequenceGroups.initialize() then indexed the *original* stop_sequences list by the *filtered* list's index. Every entry after a dropped one shifted by one position, so a group's sequence_str came from the wrong stop entry. Two user-visible failures: stop=["", "END"] -> to_strings() == [] "END" silently loses string matching entirely. stop=["unknown", "stop2"] -> to_strings() == ["unknown"] generation stops on a string the user never requested, because "unknown"'s string got attached to "stop2"'s token ids. The second is the damaging one: these strings drive stop matching in DecodeReq.stop_sequences_str_match() and trailing-stop trimming in the OpenAI completion path, so a request can terminate early on unrelated text. Carry (token_ids, original_entry) pairs through the filter so the two can never drift apart. stop_sentences_to_token_ids() keeps its original signature as a thin wrapper over the new helper. Also repairs the test module, which imported a DecodeNode class that no longer exists in sampling_params and therefore failed at collection -- meaning none of these tests had been running. Replaced with an equivalent NodeUUId round-trip test and added regression coverage for the alignment bug above. Co-Authored-By: Claude Opus 5 --- lightllm/server/core/objs/sampling_params.py | 27 +++++++---- .../server/core/objs/test_sampling_params.py | 47 ++++++++++++------- 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/lightllm/server/core/objs/sampling_params.py b/lightllm/server/core/objs/sampling_params.py index 8e31c50624..623bd52d30 100644 --- a/lightllm/server/core/objs/sampling_params.py +++ b/lightllm/server/core/objs/sampling_params.py @@ -66,31 +66,42 @@ def initialize(self, stop_sequences: Union[str, List[Union[List[int], str]]], to elif isinstance(stop_sequences, str): stop_sequences = [stop_sequences] - groups: List[List[int]] = self.stop_sentences_to_token_ids(stop_sequences, tokenizer) + # 这里必须使用 (token_ids, 原始条目) 的配对结果：空条目（如 "" 或 []）会被过滤掉， + # 若还按原始下标去取 stop_sequences[group_idx]，后面的条目就会和 token id 组错位， + # 导致 sequence_str 挂到别的组上（丢失停止字符串，或凭空多出一个停止字符串）。 + groups: List[Tuple[List[int], Union[List[int], str]]] = self._stop_sentences_to_token_id_pairs( + stop_sequences, tokenizer + ) self.size = len(groups) assert self.size <= MAX_STOP_SEQUENCES, "Too many stop sequence groups." - for group_idx in range(self.size): - if isinstance(stop_sequences[group_idx], str): - self.groups[group_idx].initialize(groups[group_idx], sequence_str=stop_sequences[group_idx]) + for group_idx, (token_ids, stop_info) in enumerate(groups): + if isinstance(stop_info, str): + self.groups[group_idx].initialize(token_ids, sequence_str=stop_info) else: - self.groups[group_idx].initialize(groups[group_idx]) + self.groups[group_idx].initialize(token_ids) - def stop_sentences_to_token_ids(self, stop_sequences: List[Union[List[int], str]], tokenizer) -> List[List[int]]: + def _stop_sentences_to_token_id_pairs( + self, stop_sequences: List[Union[List[int], str]], tokenizer + ) -> List[Tuple[List[int], Union[List[int], str]]]: + """返回 (token_ids, 原始 stop 条目) 的列表，保证两者一一对应。""" new_stop_sequences = [] for stop_info in stop_sequences: if isinstance(stop_info, str): stop_str_ids = self._stop_str_to_token_ids(stop_info, tokenizer) if stop_str_ids is not None and len(stop_str_ids) > 0: - new_stop_sequences.append(stop_str_ids) + new_stop_sequences.append((stop_str_ids, stop_info)) if isinstance(stop_info, list): if all(isinstance(x, int) for x in stop_info): if len(stop_info) > 0: - new_stop_sequences.append(stop_info) + new_stop_sequences.append((stop_info, stop_info)) else: assert False, "stop_sequences item must be type List[int] when it is a list." return new_stop_sequences + def stop_sentences_to_token_ids(self, stop_sequences: List[Union[List[int], str]], tokenizer) -> List[List[int]]: + return [token_ids for token_ids, _ in self._stop_sentences_to_token_id_pairs(stop_sequences, tokenizer)] + def _stop_str_to_token_ids(self, stop_str: str, tokenizer) -> List[int]: stop_str_ids = tokenizer.encode(stop_str, add_special_tokens=False) return stop_str_ids diff --git a/unit_tests/server/core/objs/test_sampling_params.py b/unit_tests/server/core/objs/test_sampling_params.py index ef3f08d2fc..a1c59e1564 100644 --- a/unit_tests/server/core/objs/test_sampling_params.py +++ b/unit_tests/server/core/objs/test_sampling_params.py @@ -5,10 +5,10 @@ RegularConstraint, AllowedTokenIds, ExponentialDecayLengthPenalty, - DecodeNode, SamplingParams, GuidedGrammar, GuidedJsonSchema, + NodeUUId, STOP_SEQUENCE_MAX_LENGTH, REGULAR_CONSTRAINT_MAX_LENGTH, ALLOWED_TOKEN_IDS_MAX_LENGTH, @@ -117,22 +117,35 @@ def test_exponential_decay_length_penalty_initialization(): penalty.initialize((5, 0.5)) -def test_decode_node_initialization(): - node = DecodeNode() - data = { - "node_id": 12345678901234567890, # 示例 UUID - "ip": "192.168.1.1", - "rpyc_port": 8080, - "max_new_tokens": 10, - } - node.initialize(data) - assert node.exists is True - assert node.node_id.node_id_high == (12345678901234567890 >> 64) & 0xFFFFFFFFFFFFFFFF - assert node.node_id.node_id_low == 12345678901234567890 & 0xFFFFFFFFFFFFFFFF - assert node.ip[0] == 192 - assert node.ip[1] == 168 - assert node.ip[2] == 1 - assert node.ip[3] == 1 +def test_node_uuid_roundtrip(): + node_id = 12345678901234567890 + uuid = NodeUUId() + uuid.initialize(node_id) + assert uuid.node_id_high == (node_id >> 64) & 0xFFFFFFFFFFFFFFFF + assert uuid.node_id_low == node_id & 0xFFFFFFFFFFFFFFFF + assert uuid.get() == node_id + + +@pytest.mark.parametrize( + "stop_sequences, expected_token_ids, expected_strings", + [ + # 全部有效，无过滤，token id 组与字符串一一对应。 + (["stop1", "stop2"], [[1, 2], [3, 4]], ["stop1", "stop2"]), + # 前置的空字符串会被过滤掉，后面的条目不能因此错位。 + (["", "stop1"], [[1, 2]], ["stop1"]), + # 前置的空 id 列表同理。 + ([[], "stop2"], [[3, 4]], ["stop2"]), + # 被过滤掉的字符串条目不能把自己的字符串挂到后一个条目的 token id 上。 + (["unknown", "stop2"], [[3, 4]], ["stop2"]), + # 纯 id 条目不携带字符串。 + ([[7, 8], "stop1"], [[7, 8], [1, 2]], ["stop1"]), + ], +) +def test_stop_sequence_groups_keeps_ids_and_strings_aligned(stop_sequences, expected_token_ids, expected_strings): + groups = StopSequenceGroups() + groups.initialize(stop_sequences, MockTokenizer()) + assert groups.to_list() == expected_token_ids + assert sorted(groups.to_strings()) == sorted(expected_strings) def test_sampling_params_initialization(): From 66f6a4acdf3b0180797ed1e2485cec310e37ab48 Mon Sep 17 00:00:00 2001 From: divyanarahari97 Date: Mon, 10 Aug 2026 12:25:39 -0700 Subject: [PATCH 2/2] fix(sampling): validate allowed_token_ids input instead of the empty buffer AllowedTokenIds.initialize() asserted over self.ids -- the ctypes array being written into, which at that point is still zero-filled -- rather than the incoming ids argument. Iterating a c_int array always yields Python ints, so the assertion was vacuously true and never rejected anything. Non-int input therefore fell through to the slice assignment on the next line and surfaced as a raw ctypes TypeError ("'str' object cannot be interpreted as an integer") instead of the intended AssertionError with its message. Matches the equivalent check in StopSequence.initialize, which correctly validates its argument. Co-Authored-By: Claude Opus 5 --- lightllm/server/core/objs/sampling_params.py | 2 +- unit_tests/server/core/objs/test_sampling_params.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lightllm/server/core/objs/sampling_params.py b/lightllm/server/core/objs/sampling_params.py index 623bd52d30..25ac7f26fd 100644 --- a/lightllm/server/core/objs/sampling_params.py +++ b/lightllm/server/core/objs/sampling_params.py @@ -213,7 +213,7 @@ class AllowedTokenIds(ctypes.Structure): def initialize(self, ids: List[int]): self.size = len(ids) assert self.size <= ALLOWED_TOKEN_IDS_MAX_LENGTH, "Too many allowed token IDs." - assert all(isinstance(e, int) for e in self.ids), "all must be int" + assert all(isinstance(e, int) for e in ids), "all must be int" self.ids[: self.size] = ids[:] def to_list(self): diff --git a/unit_tests/server/core/objs/test_sampling_params.py b/unit_tests/server/core/objs/test_sampling_params.py index a1c59e1564..9ba46c1526 100644 --- a/unit_tests/server/core/objs/test_sampling_params.py +++ b/unit_tests/server/core/objs/test_sampling_params.py @@ -126,6 +126,12 @@ def test_node_uuid_roundtrip(): assert uuid.get() == node_id +def test_allowed_token_ids_rejects_non_int(): + allowed_ids = AllowedTokenIds() + with pytest.raises(AssertionError): + allowed_ids.initialize([1, "2", 3]) + + @pytest.mark.parametrize( "stop_sequences, expected_token_ids, expected_strings", [