Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/server/vllm_options.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,8 @@ def engine_args(is_lora: bool, sleep_mode: bool = False) -> dict[str, Any]:
# A checkpoint's generation_config.json could add sampling params the client never asked for, such as a repetition penalty.
args["generation_config"] = "vllm"
args["max_model_len"] = int(os.getenv("VLLM_MAX_MODEL_LEN", "8192"))
# With vLLM's default of 20, the runner also computes a top-20 over the vocabulary on every decode step which is not needed for OpenRL.
args["max_logprobs"] = 0
# vLLM's own default of 0.92 has run FFT delta applies out of memory on an L4.
args["gpu_memory_utilization"] = gpu_memory_utilization()
args.update(sampler_batch_limits())
Expand Down
5 changes: 5 additions & 0 deletions tests/test_sampler_engine_kwargs.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ def test_the_tpu_platform_captures_no_cuda_graphs(self) -> None:
self.assertNotIn("compilation_config", args(False, on_tpu=True))
self.assertIn("compilation_config", args(False, on_tpu=False))

def test_no_sampler_computes_top_logprobs(self) -> None:
for is_lora, on_tpu in ((False, False), (False, True), (True, False), (True, True)):
with self.subTest(is_lora=is_lora, on_tpu=on_tpu):
self.assertEqual(args(is_lora, on_tpu)["max_logprobs"], 0)

def test_the_tpu_platform_is_out_of_tree_with_tpu_devices(self) -> None:
cases = (((True, "tpu"), True), ((True, "cuda"), False), ((False, "tpu"), False), ((False, "cuda"), False))
for (out_of_tree, device_type), expected in cases:
Expand Down
Loading