From 201475158e4dccc6717404315d2c94c10595b686 Mon Sep 17 00:00:00 2001 From: h-guo18 <67671475+h-guo18@users.noreply.github.com> Date: Mon, 24 Aug 2026 03:43:42 +0000 Subject: [PATCH] fix(launcher): use the schema's draft_model global var, and validate the schema `pipeline.global_vars` is a fixed-field dataclass (`GlobalVariables` in tools/launcher/core.py), not a free-form mapping, so an unknown key fails at launch with "No parameter named 'X' exists". The Nemotron-3.5 DSpark warm-start example (#2149) invented `drafter:`, so the example in the repo could not run at all: Error processing argument 'pipeline.global_vars.drafter=...': Invalid argument: No parameter named 'drafter' exists for `draft_model` is the field that already exists for exactly this purpose. Renaming the key and its one reference fixes the example. The Kimi-K2.5 specdec_bench example had the same latent break with `draft_model_dir:`; only the global-var key is renamed there, the script's `--draft_model_dir` flag is unchanged. This is the second time the class has shipped -- the comment on `GlobalVariables.draft_model` records the first (OMNIML-5024) -- so check_launcher_yaml now rejects unknown global_vars keys and references to keys that are never defined. It reads the field names out of core.py with a regex rather than importing it, since importing pulls in nemo_run, which the pre-commit environment does not have. Verified both ways: the check passes on every launcher YAML in the tree, and reintroducing `drafter:` reproduces the error as a pre-commit failure naming the valid keys. Signed-off-by: h-guo18 <67671475+h-guo18@users.noreply.github.com> --- .../moonshotai/Kimi-K2.5/specdec_bench.yaml | 6 +-- .../hf_streaming_dspark_warmstart.yaml | 4 +- tools/precommit/check_launcher_yaml.py | 48 +++++++++++++++++++ 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/tools/launcher/examples/moonshotai/Kimi-K2.5/specdec_bench.yaml b/tools/launcher/examples/moonshotai/Kimi-K2.5/specdec_bench.yaml index a25a3fe3452..d333a606cc8 100644 --- a/tools/launcher/examples/moonshotai/Kimi-K2.5/specdec_bench.yaml +++ b/tools/launcher/examples/moonshotai/Kimi-K2.5/specdec_bench.yaml @@ -32,13 +32,13 @@ pipeline: global_vars: hf_model: /hf-local/nvidia/Kimi-K2.5-NVFP4 - # Trained+exported DFLASH draft; override: pipeline.global_vars.draft_model_dir= - draft_model_dir: /hf-local/nvidia/Kimi-K2.5-DFlash + # Trained+exported DFLASH draft; override: pipeline.global_vars.draft_model= + draft_model: /hf-local/nvidia/Kimi-K2.5-DFlash task_0: script: common/specdec_bench/run.sh args: - - --draft_model_dir <> + - --draft_model_dir <> - --speculative_algorithm DFLASH - --engine VLLM - --mtbench /hf-local/HuggingFaceH4/mt_bench_prompts/raw/question.jsonl diff --git a/tools/launcher/examples/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/hf_streaming_dspark_warmstart.yaml b/tools/launcher/examples/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/hf_streaming_dspark_warmstart.yaml index ddf17ba01f1..9b2334b2dc0 100644 --- a/tools/launcher/examples/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/hf_streaming_dspark_warmstart.yaml +++ b/tools/launcher/examples/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/hf_streaming_dspark_warmstart.yaml @@ -56,7 +56,7 @@ pipeline: # config.json layer-type vocabulary already patched for tf5 (see header). hf_model: /hf-local/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 # The published drafter the warm start continues from. - drafter: /hf-local/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DSpark + draft_model: /hf-local/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DSpark # Build /scratchspace/data/train.jsonl. Point data.data_path at the full # Spec-Decoding-Dataset-v2 corpus to reproduce; eagle_utils also accepts a @@ -80,7 +80,7 @@ pipeline: # causal attention and attention sink all come from this recipe — see header. - --config modules/Model-Optimizer/modelopt_recipes/huggingface/models/nvidia/Nemotron-3.5-Lightning-30B-A3B-BF16/speculative_decoding/dspark_warmstart.yaml - model.model_name_or_path=<> - - dflash.dflash_init_checkpoint=<> + - dflash.dflash_init_checkpoint=<> - data.data_path=/scratchspace/data/train.jsonl # The stock Nemotron template has no {% generation %} tags; without a tagged copy # answer_only_loss trains on an all-zero mask (see header). diff --git a/tools/precommit/check_launcher_yaml.py b/tools/precommit/check_launcher_yaml.py index 2e0f8f92617..cc6f052cb88 100644 --- a/tools/precommit/check_launcher_yaml.py +++ b/tools/precommit/check_launcher_yaml.py @@ -112,6 +112,52 @@ def _try_load_recipe(recipe_path: Path, source: Path) -> list[str]: return [] +def _global_vars_schema() -> set[str] | None: + """Field names accepted by ``GlobalVariables``, or None if it can't be read. + + Parsed out of ``core.py`` rather than imported: importing it pulls in ``nemo_run``, + which is not a dependency of the pre-commit environment. + """ + core = _LAUNCHER_DIR / "core.py" + try: + source = core.read_text(encoding="utf-8") + except OSError: + return None + match = re.search(r"^class GlobalVariables.*?(?=^@|\Z)", source, re.MULTILINE | re.DOTALL) + if not match: + return None + return set(re.findall(r"^\s{4}(\w+)\s*:", match.group(0), re.MULTILINE)) + + +def _check_global_vars(pipeline: dict, path: Path) -> list[str]: + """Reject ``global_vars`` keys the launcher's dataclass cannot accept. + + ``global_vars`` is a fixed-field dataclass, not a free-form mapping, so an unknown key + fails at launch with ``No parameter named 'X' exists`` — after the user has set up a + cluster environment. This has now bitten twice (OMNIML-5024, then the Nemotron-3.5 + DSpark warm-start example), so it is checked here instead. + """ + schema = _global_vars_schema() + global_vars = pipeline.get("global_vars") + if schema is None or not isinstance(global_vars, dict): + return [] + errors = [ + f"{path}: global_vars key {key!r} is not a field of GlobalVariables " + f"(valid: {', '.join(sorted(schema))})" + for key in global_vars + if key not in schema + ] + # A reference to a key that is never defined interpolates to the literal + # ``<>`` and reaches the job as a nonsense path. + refs = sorted(set(re.findall(r"<>", path.read_text("utf-8")))) + errors.extend( + f"{path}: <> is referenced but never defined" + for ref in refs + if ref not in global_vars + ) + return errors + + def _scan_launcher_yaml(path: Path) -> list[str]: errors: list[str] = [] try: @@ -124,6 +170,8 @@ def _scan_launcher_yaml(path: Path) -> list[str]: if not isinstance(pipeline, dict): return [] + errors.extend(_check_global_vars(pipeline, path)) + for task in pipeline.values(): if not isinstance(task, dict): continue