From f0f9ae1bd8ec646ea2419d479ca1513702270b4d Mon Sep 17 00:00:00 2001 From: NopeCharon <115794254+NopeCharon@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:36:00 +0000 Subject: [PATCH] feat(workers): read HF_TOKEN from an optional hf-token Secret Workers that the scheduler starts had no Hugging Face token, so a job on a gated model (Gemma 3, Llama) could not download its weights on a cluster, and every worker downloaded public models anonymously, under the Hub's lower rate limits. The static worker Deployments used to read HF_TOKEN from an optional `hf-token` Secret; that went away with them when the bundles moved to the scheduler (#277), and the pod template the API server renders never picked it up. pod_env() now gives every trainer and sampler container HF_TOKEN from the `token` key of the `hf-token` Secret in the workers' namespace, the same Secret and key the old manifests used. The reference is optional, so a cluster without the Secret starts its workers exactly as before. The token stays in the Secret: the API server's own HF_TOKEN is never copied into the Workload, where anyone who can read Workloads would see it. The one-off e2e client and vLLM eval Jobs under k8s/eval read the same Secret, since they load the same models' tokenizers and weights. docs/setup/lora-dra.md gets a "Use gated models" section with the kubectl command that creates the Secret, the GKE guide and docs/configuration.md point at it, and item 5 of TODOs.md is marked completed. Signed-off-by: NopeCharon <115794254+NopeCharon@users.noreply.github.com> --- TODOs.md | 3 ++- docs/configuration.md | 5 +++++ docs/setup/gke-setup.md | 2 +- docs/setup/lora-dra.md | 19 +++++++++++++++++++ k8s/eval/e2e-client-job.yaml | 7 +++++++ k8s/eval/vllm-eval-job.yaml | 7 +++++++ src/server/scheduler_worker_manager.py | 5 +++++ tests/test_scheduler_worker_manager.py | 14 ++++++++++++++ 8 files changed, 60 insertions(+), 2 deletions(-) diff --git a/TODOs.md b/TODOs.md index 6cb0ad084..0a4b2fa38 100644 --- a/TODOs.md +++ b/TODOs.md @@ -20,7 +20,8 @@ - **Current Behavior:** `KubernetesFFTWorkerManager` launches trainer and sampler workloads as bare Kubernetes `Pod` objects (`open-rl-trainer-...` and `open-rl-sampler-...`). When GCP Spot VM preemption reclaims an underlying GPU spot node, Kubernetes terminates the bare worker pod without respawning a replacement, causing the distributed E2E training job to hang indefinitely waiting for rollouts or gradient updates. - **Improvement:** Equip the worker provisioner with automated fault tolerance against spot preemption by either deploying worker workloads as lightweight Kubernetes `Deployment` or `ReplicaSet` controllers (with `replicas: 1`), or by implementing active pod lifecycle monitoring and automated recreation within `KubernetesFFTWorkerManager`. Upon spot preemption, Kubernetes or the provisioner will automatically schedule a replacement worker pod onto a fresh spot node. When the replacement pod boots, it will automatically resume from the latest saved model checkpoint on shared persistent storage (`/mnt/shared/open-rl/...`), ensuring uninterrupted distributed fine-tuning. -## 5. HuggingFace Hub Token (`HF_TOKEN`) Injection via Kubernetes Secret / ConfigMap +## 5. [COMPLETED] HuggingFace Hub Token (`HF_TOKEN`) Injection via Kubernetes Secret / ConfigMap +- **Status:** Implemented in `src/server/scheduler_worker_manager.py` (`pod_env()`). Every worker pod sets `HF_TOKEN` from the `token` key of the optional `hf-token` Secret in the workers' namespace; see `docs/setup/lora-dra.md` ("Use gated models"). - **Current Behavior:** Dynamically spawned `Trainer` and `Sampler` worker containers run without authentication (`user_id=public`), causing strict HuggingFace Hub CDN throttling (`403 Forbidden` / lengthy retry loops on `xet_client` during first-time model caching) and failing on gated models (`e.g., Gemma 2, Gemma 4, Llama 3`). - **Improvement:** Update `scheduler_worker_manager.py` (`render_template()`) and static Kubernetes manifests to inject an optional `open-rl-hf-secret` (`or ConfigMap via envFrom / set_env`) into all worker containers, ensuring authenticated, high-bandwidth model downloads without rate-limiting across multi-pod cluster setups. diff --git a/docs/configuration.md b/docs/configuration.md index e239e3d63..177c59873 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -102,6 +102,11 @@ each trainer and sampler, and the OpenRL scheduler starts their pods. The API server writes each worker's environment into the `Workload`'s pod template, so there are no worker deployments to configure by hand. +Workers read `HF_TOKEN` from the `token` key of the Secret `hf-token` in their +namespace, if it exists. Gated Hugging Face models need it; see +[Use gated models](setup/lora-dra.md#use-gated-models). The API server does not +pass its own `HF_TOKEN` to the workers. + ## Sampling pools Sampling goes through sampling pools. A pool serves one FFT job, one exclusive diff --git a/docs/setup/gke-setup.md b/docs/setup/gke-setup.md index 1c574cb8f..14c7e77ca 100644 --- a/docs/setup/gke-setup.md +++ b/docs/setup/gke-setup.md @@ -127,7 +127,7 @@ Apply **only one** of the following release bundles. Both install the same API s ``` The DaemonSets run privileged, with host PID and host network, and schedule onto nodes labeled `nvidia.com/gpu.present=true` (set on the pool above). The snapshot agent checkpoints CUDA processes and needs NVIDIA driver r570 or newer, which the `latest` driver installer above provides. FFT workers never share a GPU with a LoRA worker. See [GKE FFT Time-Slice Setup](gke-fft-timeslice.md) for how the sharing works. -Replace `latest/download` with `download/` to pin a specific version. See [OpenRL on an existing DRA cluster](lora-dra.md) for what each bundle installs, and for using storage other than Filestore. +Replace `latest/download` with `download/` to pin a specific version. See [OpenRL on an existing DRA cluster](lora-dra.md) for what each bundle installs, for using gated Hugging Face models, and for using storage other than Filestore. To track unreleased code on `main` instead, render the overlay from a checkout, `k8s/deploy/lora` or `k8s/deploy/fft`: diff --git a/docs/setup/lora-dra.md b/docs/setup/lora-dra.md index 63711fe6c..1e5876672 100644 --- a/docs/setup/lora-dra.md +++ b/docs/setup/lora-dra.md @@ -96,6 +96,25 @@ Any Tinker SDK from 0.23 onward can talk to the API server. SDKs from 0.25 send training requests and read training and sampling results as protobuf; the API server serves both that and the older JSON encoding. +## Use gated models + +Workers download weights from Hugging Face, and gated models such as Gemma 3 or +Llama need a token from an account that has accepted the model's license. A +token also gives public models the higher download rate limits of a signed-in +account. Store it in a Secret named `hf-token`, under the key `token`, in the +workers' namespace: + +```bash +kubectl --context my-cluster -n openrl-system create secret generic hf-token \ + --from-literal=token="$HF_TOKEN" +``` + +Every trainer and sampler pod sets `HF_TOKEN` from it. The reference is +optional, so without the Secret workers start as before and load public models +only. A worker reads the Secret when its pod starts, so after creating or +changing it, workers that are already running keep the token they started with +until they are replaced. + ## Use different storage Both bundles create one PersistentVolumeClaim, `open-rl-shared-pvc`, on the GKE diff --git a/k8s/eval/e2e-client-job.yaml b/k8s/eval/e2e-client-job.yaml index 7bd750358..b9491eb39 100644 --- a/k8s/eval/e2e-client-job.yaml +++ b/k8s/eval/e2e-client-job.yaml @@ -44,6 +44,13 @@ spec: value: "1" - name: HF_HOME value: "/mnt/shared/open-rl/huggingface" + # Gated models' tokenizers and weights; the workers read the same Secret. + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token + key: token + optional: true resources: limits: memory: "16Gi" diff --git a/k8s/eval/vllm-eval-job.yaml b/k8s/eval/vllm-eval-job.yaml index b944d16f4..622e21190 100644 --- a/k8s/eval/vllm-eval-job.yaml +++ b/k8s/eval/vllm-eval-job.yaml @@ -53,6 +53,13 @@ spec: value: "/mnt/shared/open-rl/evals/last-eval.json" - name: HF_HOME value: "/mnt/shared/open-rl/huggingface" + # Gated models' tokenizers and weights; the workers read the same Secret. + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token + key: token + optional: true resources: claims: - name: trainer-gpu diff --git a/src/server/scheduler_worker_manager.py b/src/server/scheduler_worker_manager.py index a6586986b..6f9e25770 100644 --- a/src/server/scheduler_worker_manager.py +++ b/src/server/scheduler_worker_manager.py @@ -65,6 +65,8 @@ # On a pool's Deployment and NetworkPolicy, so release_owner finds them after the pool's Workloads are gone. OWNER_LABEL = "openrl.io/owner" PLACEMENT_HISTORY, PLACEMENT_HISTORY_SECONDS = 200, 7 * 24 * 3600 +# The Secret in the workers' namespace whose `token` key workers read as HF_TOKEN. +HF_TOKEN_SECRET = "hf-token" def workload_name(role: str, owner: str, is_lora: bool, index: int = 0) -> str: @@ -175,6 +177,9 @@ def pod_env(worker: Worker) -> list[dict[str, Any]]: env.append(downward("POD_IP", "status.podIP")) # The pod a worker names on the records it keeps. env += [downward("POD_NAME", "metadata.name"), downward("POD_UID", "metadata.uid"), downward("NODE_NAME", "spec.nodeName")] + # Gated models need a Hugging Face token. It stays in its Secret rather than in the Workload, and a cluster + # without the Secret starts its workers without one. + env.append({"name": "HF_TOKEN", "valueFrom": {"secretKeyRef": {"name": HF_TOKEN_SECRET, "key": "token", "optional": True}}}) return env diff --git a/tests/test_scheduler_worker_manager.py b/tests/test_scheduler_worker_manager.py index d5abafe9e..7fb8d0a7e 100644 --- a/tests/test_scheduler_worker_manager.py +++ b/tests/test_scheduler_worker_manager.py @@ -186,6 +186,20 @@ def test_workers_get_the_deployment_settings(self) -> None: env = {e["name"]: e.get("value") for e in pod["spec"]["template"]["spec"]["containers"][0]["env"]} self.assertEqual({k: env.get(k) for k in settings}, settings) + def test_workers_read_the_hugging_face_token_from_its_secret(self) -> None: + s = self.store_with("job-lora-1", {"base_model": "google/gemma-3-1b-it", "fine_tuning_type": "lora"}) + # A token the API server holds is not copied into the Workload, where anyone who can read Workloads would see it. + with patch.dict(os.environ, {"HF_TOKEN": "hf_server_token"}), patch("server.worker_manager.get_state_store", return_value=s): + self.manager.ensure("job-lora-1", "trainer") + self.manager.ensure("job-lora-1", "sampler") + + self.assertEqual(len(self.api.created), 2) + for pod in self.api.created: + self.assertNotIn("hf_server_token", json.dumps(pod)) + (token,) = [e for e in pod["spec"]["template"]["spec"]["containers"][0]["env"] if e["name"] == "HF_TOKEN"] + # Optional, so workers on a cluster without the Secret still start. + self.assertEqual(token, {"name": "HF_TOKEN", "valueFrom": {"secretKeyRef": {"name": "hf-token", "key": "token", "optional": True}}}) + def test_samplers_keep_the_vllm_compile_cache_on_the_shared_volume(self) -> None: s = self.store_with("job-lora-1", {"base_model": "Qwen/Qwen2.5-0.5B", "fine_tuning_type": "lora"}) with patch("server.worker_manager.get_state_store", return_value=s):