Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -134,9 +134,7 @@ Cluster setup, node pools and the per-model knobs are in
[OpenRL on GKE TPU nodes](setup/tpu-on-gke.md).

The TPU sampler image installs the released `vllm-torchtpu` wheel, which serves
full fine-tuning but not LoRA: multi-LoRA support is not in a release yet, so a
TPU sampler image for LoRA jobs has to be built from vllm-torchtpu's main
branch.
both LoRA and full fine-tuning.

In scheduler mode these variables on the API server set up the worker pods:

Expand Down
4 changes: 1 addition & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,10 @@ tpu = [
"transformers>=5.5.4",
]
# TPU sampler: vLLM for TPU from PyPI (torch-tpu, CPU torch, JAX and libtpu).
# TODO: multi-LoRA on TPU is not in this release; it needs vllm-torchtpu built
# from main with PRs #2376, #2359, #2378 and #2360 until a release carries them.
tpu-sampler = [
"torch==2.13.0; sys_platform == 'linux' and platform_machine == 'x86_64'",
"torchvision==0.28.0; sys_platform == 'linux' and platform_machine == 'x86_64'",
"vllm-torchtpu==0.32.0; sys_platform == 'linux' and platform_machine == 'x86_64'",
"vllm-torchtpu==0.33.0; sys_platform == 'linux' and platform_machine == 'x86_64'",
]
# Kubernetes cluster mode: the k8s API client used to launch FFT worker pods.
cluster = [
Expand Down
7 changes: 3 additions & 4 deletions src/server/Dockerfile.tpu_sampler
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,9 @@

# TPU sampler image. Same layout as src/server/Dockerfile.tpu, with the
# `tpu-sampler` extra (vllm-torchtpu: vLLM for TPU on torch-tpu, CPU torch,
# JAX and libtpu) instead of `tpu`. The model stays torch Parameters, so a full
# checkpoint can be reloaded in place each step.
# TODO: multi-LoRA on TPU is not in this release; it needs vllm-torchtpu built
# from main with PRs #2376, #2359, #2378 and #2360 until a release carries them.
# JAX and libtpu) instead of `tpu`. It serves LoRA and full fine-tuning; the
# model stays torch Parameters, so a full checkpoint can be reloaded in place
# each step.

FROM ubuntu:22.04

Expand Down
8 changes: 2 additions & 6 deletions src/server/tpu_worker_extension.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
"""Weight updates for the TPU platform's worker, mixed in through vLLM's worker_extension_cls.

The TPU worker has none of the GPU worker's weight-transfer methods, nor
reset_encoder_cache, which the sampler calls after each update. These mirror
vLLM's GPU worker for one model with no draft model.
The TPU worker has none of the GPU worker's weight-transfer methods. These
mirror vLLM's GPU worker for one model with no draft model.
"""

from typing import Any
Expand All @@ -17,9 +16,6 @@
class TPUWeightTransferExtension:
_open_rl_transfer_engine: WeightTransferEngine | None = None

def reset_encoder_cache(self) -> None:
self.model_runner.reset_encoder_cache()

def _open_rl_engine(self) -> WeightTransferEngine:
# Created on first use: the TPU worker builds no engine at load time and keeps no self.device.
if self._open_rl_transfer_engine is None:
Expand Down
5 changes: 0 additions & 5 deletions tests/test_tpu_worker_extension.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,6 @@ def test_a_model_without_the_flipped_flag_fails_before_any_update(self) -> None:
FakeTPUWorker(object(), flipped=False).start_weight_update()
self.create_engine.assert_not_called()

def test_reset_encoder_cache_goes_to_the_model_runner(self) -> None:
worker = FakeTPUWorker(object())
worker.reset_encoder_cache()
worker.model_runner.reset_encoder_cache.assert_called_once_with()


if __name__ == "__main__":
unittest.main()
Loading
Loading