From 5c563032dca98f9c9a02ca122df27fd17bb06e2a Mon Sep 17 00:00:00 2001 From: Hoda Mohammadi Date: Thu, 19 Nov 2026 10:31:14 -0800 Subject: [PATCH] perf(sampler): set max_logprobs to 0 Signed-off-by: Hoda Mohammadi --- src/server/vllm_options.py | 2 ++ tests/test_sampler_engine_kwargs.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/src/server/vllm_options.py b/src/server/vllm_options.py index 6f1f68a4..26c375d8 100644 --- a/src/server/vllm_options.py +++ b/src/server/vllm_options.py @@ -145,6 +145,8 @@ def engine_args(is_lora: bool, sleep_mode: bool = False) -> dict[str, Any]: # A checkpoint's generation_config.json could add sampling params the client never asked for, such as a repetition penalty. args["generation_config"] = "vllm" args["max_model_len"] = int(os.getenv("VLLM_MAX_MODEL_LEN", "8192")) + # With vLLM's default of 20, the runner also computes a top-20 over the vocabulary on every decode step which is not needed for OpenRL. + args["max_logprobs"] = 0 # vLLM's own default of 0.92 has run FFT delta applies out of memory on an L4. args["gpu_memory_utilization"] = gpu_memory_utilization() args.update(sampler_batch_limits()) diff --git a/tests/test_sampler_engine_kwargs.py b/tests/test_sampler_engine_kwargs.py index a63ffd62..0e070321 100644 --- a/tests/test_sampler_engine_kwargs.py +++ b/tests/test_sampler_engine_kwargs.py @@ -37,6 +37,11 @@ def test_the_tpu_platform_captures_no_cuda_graphs(self) -> None: self.assertNotIn("compilation_config", args(False, on_tpu=True)) self.assertIn("compilation_config", args(False, on_tpu=False)) + def test_no_sampler_computes_top_logprobs(self) -> None: + for is_lora, on_tpu in ((False, False), (False, True), (True, False), (True, True)): + with self.subTest(is_lora=is_lora, on_tpu=on_tpu): + self.assertEqual(args(is_lora, on_tpu)["max_logprobs"], 0) + def test_the_tpu_platform_is_out_of_tree_with_tpu_devices(self) -> None: cases = (((True, "tpu"), True), ((True, "cuda"), False), ((False, "tpu"), False), ((False, "cuda"), False)) for (out_of_tree, device_type), expected in cases: