diff --git a/KLAUD_DEBUG.md b/KLAUD_DEBUG.md index e24a5832c9..ff25ddea4e 100644 --- a/KLAUD_DEBUG.md +++ b/KLAUD_DEBUG.md @@ -402,3 +402,14 @@ are skipped, and registry `/` and enroot `#` image spellings are normalized for historical identity matching and point backfill. A transient or malformed baseline for one candidate defers that candidate and continues through the reviewed pool; it does not consume or block later candidate slots. + +## 12. Qwen3.8-27B native MTP: FlashInfer autotune startup + +The pinned CUDA image can fail its optional MTP dummy-prefill autotune with +`scheduler_metadata must have shape (metadata_size)` in FlashAttention on Hopper. +Use the supported `--no-enable-flashinfer-autotune` recipe option. See the +[English procedure](docs/configuration-procedures.md#qwen38-27b-fp8-native-mtp-golden-acceptance) +and [中文说明](docs/configuration-procedures_zh.md#qwen38-27b-fp8-原生-mtp-黄金接受长度) +for the preserved native draft precision and configuration. +[Failure](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35495730774); +[passing H100 concurrency-1 validation](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35496528224). diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.827b_fp8_h100_vllm_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.827b_fp8_h100_vllm_mtp.sh new file mode 100755 index 0000000000..db7fca1cdd --- /dev/null +++ b/benchmarks/single_node/fixed_seq_len/qwen3.827b_fp8_h100_vllm_mtp.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Qwen3.8-27B-FP8 (fp8 e4m3 dynamic-activation checkpoint of the dense hybrid +# attention model: 48 linear-attention and 16 full attention layers) on one +# H100, served by vLLM with the original BF16 native MTP head (one MTP layer, +# mtp_num_hidden_layers=1) drafting three tokens per step. +# https://recipes.vllm.ai/Qwen/Qwen3.8-27B +# https://huggingface.co/Qwen/Qwen3.8-27B-FP8 +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC ISL OSL RANDOM_RANGE_RATIO RESULT_FILENAME \ + MAX_MODEL_LEN EVAL_ONLY RUN_EVAL THINKING_MODE + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + check_env_vars SLURMD_NODENAME + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +if [[ "$TP" -ne 1 ]]; then + echo "This recipe serves Qwen3.8-27B-FP8 on a single GPU; got TP=$TP" >&2 + exit 1 +fi + +# The three-token recipe uses the measured thinking-on curve from PR #3304. +# Qwen's pinned chat template enables thinking unless explicitly overridden. +if [[ "$THINKING_MODE" != "thinking_on" ]]; then + echo "This recipe requires thinking_on to match its chat template and golden AL" >&2 + exit 1 +fi +NUM_SPEC_TOKENS=3 +TARGET_REVISION=017b9c7af6b5689d5dd426a76e0bc077eb5ca20a +DRAFT_MODEL=Qwen/Qwen3.8-27B +DRAFT_REVISION=1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0 + +nvidia-smi + +# Complete/resume partial downloads instead of trusting nonempty directories. +if [[ "$MODEL" != /* ]]; then hf download "$MODEL" --revision "$TARGET_REVISION"; fi + +SERVER_LOG=/workspace/server.log + +# Serve the matrix context (isl + osl + slack), not the checkpoint's 262K; +# accuracy evals use the eval context benchmark_lib derives. +MODEL_LEN="$MAX_MODEL_LEN" +if [[ "$EVAL_ONLY" == true ]]; then + setup_eval_context + MODEL_LEN="$EVAL_MAX_MODEL_LEN" +fi + +# vLLM's default max-num-seqs (1024) exceeds the GDN/Mamba cache blocks that fit +# next to the weights on the smaller cards (472 next to the bf16 weights on an +# 80 GB H100, run 35357364404) and engine start aborts before graph capture. +# Size the scheduler batch to the sweep point instead; the accuracy eval serves +# up to 256 concurrent requests. +MAX_NUM_SEQS=$(( CONC > 16 ? CONC : 16 )) +if [[ "$EVAL_ONLY" == true ]]; then + MAX_NUM_SEQS=256 +fi + +# Pyxis shares the host network; port 8888 can already belong to a host service. +select_available_server_port + +# Read the measured AL; original BF16 MTP modules must not inherit target FP8. +# Accuracy paths, including combined throughput+eval, use real verification. +MTP_CONFIG_DIR=$(mktemp -d /tmp/inferencex-qwen-mtp.XXXXXX) +python3 -m infx.bench_serving.qwen_mtp \ + --target-model "$MODEL" --target-revision "$TARGET_REVISION" \ + --draft-model "$DRAFT_MODEL" --draft-revision "$DRAFT_REVISION" \ + --tokens "$NUM_SPEC_TOKENS" --thinking-mode "$THINKING_MODE" \ + --eval-only "$EVAL_ONLY" --run-eval "$RUN_EVAL" \ + --golden-file "$INFERENCEX_REPO_ROOT/golden_al_distribution/qwen3.827b_fp8_mtp.yaml" \ + --output-dir "$MTP_CONFIG_DIR" +SPEC_CONFIG=$(cat "$MTP_CONFIG_DIR/speculative-config.json") +HF_OVERRIDES=$(cat "$MTP_CONFIG_DIR/hf-overrides.json") + +start_gpu_monitor + +VLLM_CMD=( + vllm serve "$MODEL" --served-model-name "$MODEL" + --revision "$TARGET_REVISION" --dtype bfloat16 + --hf-overrides "$HF_OVERRIDES" + # Avoid the pinned autotuner's FA3 scheduler-metadata error in MTP dummy prefill. + --no-enable-flashinfer-autotune + --host 0.0.0.0 --port "$PORT" + --tensor-parallel-size 1 + # Text-only serving: skip the vision tower of Qwen3_5ForConditionalGeneration. + --language-model-only + --trust-remote-code + --kv-cache-dtype fp8 + --max-model-len "$MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + # Every throughput request prefills its full random prompt; no prefix-cache hits. + --no-enable-prefix-caching + --reasoning-parser qwen3 + --enable-auto-tool-choice --tool-call-parser qwen3_xml + --speculative-config "$SPEC_CONFIG" + --disable-uvicorn-access-log +) +printf '%q ' "${VLLM_CMD[@]}" | tee /workspace/vllm_command.txt +printf '\n' | tee -a /workspace/vllm_command.txt +VLLM_LOG_MODEL_INSPECTION=1 "${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [[ "$EVAL_ONLY" == true ]]; then + run_eval --framework lm-eval --port "$PORT" + # Non-agentic evals must stage lm-eval's output into the workspace root. + append_lm_eval_summary +else + pip install -q datasets pandas + run_benchmark_serving \ + --model "$MODEL" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --use-chat-template \ + --server-pid "$SERVER_PID" + if [[ "$RUN_EVAL" == true ]]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary + fi +fi + +stop_gpu_monitor diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c5d5edac68..058491b0ce 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8375,3 +8375,22 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: dp-attn: false kv-offload-backend: name: hicache + +qwen3.827b-fp8-h100-vllm-mtp: + image: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 + model: Qwen/Qwen3.8-27B-FP8 + model-prefix: qwen3.827b + runner: cluster:h100-dgxc + precision: fp8 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 1, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 1, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 4c6abfccfa..4a4dce87bc 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -547,3 +547,31 @@ runtime directories stay out of `/workspace`. The MI300X launcher also raises it allocation from 180 to 480 minutes for this checkpoint: the HF cache there is node-local, so the first arm on each node downloads 511 GB before serving. GPU sweep and eval evidence is required before calling either arm validated. + +## Qwen3.8-27B FP8 native MTP golden acceptance + +H100 covers both 1024/1024 and 8192/1024 at TP1 and concurrency +1, 2, 4, 8, 16, 32, 64 and 128. The 8192/1024 scenario also participates in +standard eval selection; its throughput uses the same measured AL **2.52**. + +The H100, H200, MI300X and MI325X fixed-sequence FP8 recipes use three native MTP +draft tokens and the `thinking_on` curve measured in [#3304](https://github.com/SemiAnalysisAI/InferenceX/pull/3304). +Throughput-only runs load AL **2.52** from +[`qwen3.827b_fp8_mtp.yaml`](../golden_al_distribution/qwen3.827b_fp8_mtp.yaml). +The pinned Qwen chat template defaults to thinking on; these recipes reject another +`THINKING_MODE` instead of pairing a different curve with that template. The +configuration helper accepts only measured draft lengths 1–4. + +Both eval-only and combined throughput+eval runs use real standard verification. +The target remains FP8; the native head comes from the pinned original BF16 +`Qwen/Qwen3.8-27B` checkpoint with native BF16 draft KV cache. Explicit MTP module +exclusions prevent vLLM from applying target FP8 quantization to that head. +Runtime model inspection and the server command are retained in the logs. These +fixed-sequence recipes explicitly use measured synthetic throughput acceptance; +the SRT connector's non-AgentX selection policy is separate. + +The CUDA recipes disable optional FlashInfer autotuning with +`--no-enable-flashinfer-autotune`. In the pinned image, its MTP dummy prefill can +fail FlashAttention scheduler-metadata shape validation at low concurrency +([H100 evidence](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35495730774)). +This uses the supported engine option and preserves native draft precision. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index f877035f18..43edb04810 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -488,3 +488,27 @@ python -m pytest utils/matrix_logic/ -v 检查点将仓库挂载到 `/ix` 并重写 `RESULT_DIR`,使 AgentX 运行目录不落在 `/workspace` 下。MI300X launcher 还为该检查点将 Slurm 分配时长从 180 分钟提高到 480 分钟:那里的 HF 缓存为节点本地, 每个节点上的首次运行需先下载 511 GB。在获得 GPU sweep 与 eval 证据之前,不得将任一配方视为已验证。 + +## Qwen3.8-27B FP8 原生 MTP 黄金接受长度 + +H100 同时覆盖 1024/1024 和 8192/1024,均使用 TP1,并发为 +1、2、4、8、16、32、64、128。8192/1024 场景同时参与标准精度评测选择; +其吞吐测试使用相同的实测 AL **2.52**。 + +H100、H200、MI300X 和 MI325X 的固定序列长度 FP8 配方使用三个原生 MTP 草稿 token, +以及 [#3304](https://github.com/SemiAnalysisAI/InferenceX/pull/3304) 测量的 `thinking_on` 曲线。 +仅吞吐运行从 [`qwen3.827b_fp8_mtp.yaml`](../golden_al_distribution/qwen3.827b_fp8_mtp.yaml) +读取黄金 AL **2.52**。固定版本的 Qwen chat template 默认开启 thinking;这些配方会 +拒绝其他 `THINKING_MODE`,避免模板与曲线不一致。配置辅助程序只接受已测量的 +1–4 个草稿 token。 + +仅精度评测以及吞吐后紧接精度评测的运行均使用真实的标准验证。目标模型保持 FP8; +原生 MTP 头来自固定版本的原始 BF16 `Qwen/Qwen3.8-27B` checkpoint,草稿 KV cache +保持原生 BF16。显式排除 MTP 模块,防止 vLLM 将目标模型的 FP8 量化应用到草稿头。 +日志保留运行时模型检查和服务命令。这些固定序列长度配方显式使用实测的合成 +吞吐接受长度;SRT connector 的非 AgentX 选择策略另行生效。 + +CUDA 配方通过 `--no-enable-flashinfer-autotune` 关闭可选的 FlashInfer autotuning。 +固定镜像中的 MTP dummy prefill 会在低并发时触发 FlashAttention scheduler metadata +形状检查失败([H100 证据](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35495730774))。 +这是引擎支持的选项,草稿原生精度保持不变。 diff --git a/golden_al_distribution/qwen3.827b_fp8_mtp.yaml b/golden_al_distribution/qwen3.827b_fp8_mtp.yaml new file mode 100644 index 0000000000..b62e8d0e04 --- /dev/null +++ b/golden_al_distribution/qwen3.827b_fp8_mtp.yaml @@ -0,0 +1,23 @@ +# Source GitHub Actions runs: thinking_off https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35492786441; thinking_on https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35492787451 +# SPEED-Bench Qualitative coding; all 80 requests succeeded in every cell; maximum output length 4096. +# Prepared dataset SHA256: e9bbbe998472b79501ed79ba2bf126e58b63e153a6deef70ce2b63ea7a4de845 +# Target: Qwen/Qwen3.8-27B-FP8 @ 017b9c7af6b5689d5dd426a76e0bc077eb5ca20a; target weights: fp8; target KV cache: fp8. +# Image: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3; H200; TP=1; concurrency=64; context=16384; seed=0. +# Native MTP head: Qwen/Qwen3.8-27B @ 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0; original BF16 weights, compute and KV cache. +# FP8 collection excludes all original MTP weight modules from inherited target quantization through --hf-overrides. +# thinking_on: temperature=1.0, top_p=0.95, top_k=20, presence_penalty=0.0, chat_template_kwargs={"enable_thinking":true}. +# thinking_off: temperature=0.7, top_p=0.8, top_k=20, presence_penalty=1.5, chat_template_kwargs={"enable_thinking":false}. +# AL includes the bonus token and is rounded to two decimals. AR comments are fractions, using actual proposed tokens. +# Counter comments retain accepted draft tokens, proposed draft tokens, and verification drafts for exact recomputation. +# Collector: benchmarks/single_node/speedbench/qwen3.827b_vllm.sh (speedbench-al.yml); draft lengths limited to 1-4. +qwen3.8-27b-fp8: + thinking_off: + 1: 1.90 # AR=0.900382608696; accepted=25886; proposed=28750; drafts=28750 + 2: 2.68 # AR=0.837843967342; accepted=33249; proposed=39684; drafts=19842 + 3: 3.28 # AR=0.759643860940; accepted=40271; proposed=53013; drafts=17671 + 4: 3.79 # AR=0.698515476943; accepted=39807; proposed=56988; drafts=14247 + thinking_on: + 1: 1.73 # AR=0.726705818716; accepted=94043; proposed=129410; drafts=129410 + 2: 2.21 # AR=0.604951065637; accepted=119669; proposed=197816; drafts=98908 + 3: 2.52 # AR=0.507043631751; accepted=134722; proposed=265701; drafts=88567 + 4: 2.72 # AR=0.430686744384; accepted=135513; proposed=314644; drafts=78661 diff --git a/infx/bench_serving/qwen_mtp.py b/infx/bench_serving/qwen_mtp.py new file mode 100644 index 0000000000..309015192f --- /dev/null +++ b/infx/bench_serving/qwen_mtp.py @@ -0,0 +1,116 @@ +"""Prepare native BF16 MTP and measured acceptance for Qwen3.8-27B FP8 recipes.""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from typing import Any + +import yaml + +from infx.bench_serving.speedbench_acceptance import mtp_quantization_overrides + + +def build_configs( + target: dict[str, Any], + draft: dict[str, Any], + draft_weights: dict[str, str], + golden: dict[str, Any], + *, + draft_model: str, + draft_revision: str, + tokens: int, + thinking_mode: str, + eval_only: bool, + run_eval: bool, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Use synthetic acceptance only when no accuracy evaluation will run.""" + if isinstance(tokens, bool) or not 1 <= tokens <= 4: + raise ValueError("Qwen3.8-27B native MTP requires 1-4 draft tokens") + if thinking_mode not in {"thinking_on", "thinking_off"}: + raise ValueError(f"Invalid thinking mode: {thinking_mode}") + if target.get("quantization_config", {}).get("quant_method") != "fp8": + raise ValueError("The target must use the measured FP8 checkpoint") + draft_text = draft.get("text_config", draft) + if ( + draft.get("quantization_config") + or draft_text.get("dtype", draft_text.get("torch_dtype")) != "bfloat16" + ): + raise ValueError("The native MTP head must retain its original BF16 precision") + overrides = mtp_quantization_overrides(target, draft_weights) + spec: dict[str, Any] = { + "method": "mtp", + "model": draft_model, + "revision": draft_revision, + "num_speculative_tokens": tokens, + "kv_cache_dtype": "auto", + "rejection_sample_method": "standard", + } + if not (eval_only or run_eval): + try: + value = golden["qwen3.8-27b-fp8"][thinking_mode][tokens] + except (KeyError, TypeError) as error: + raise ValueError(f"Missing golden AL for {thinking_mode}, {tokens} drafts") from error + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or not 1 <= value <= tokens + 1 + ): + raise ValueError(f"Invalid golden AL: {value!r}") + spec.update(rejection_sample_method="synthetic", synthetic_acceptance_length=value) + return spec, overrides + + +def configuration_file(model: str, revision: str, filename: str) -> Path: + """Read local metadata or download it at the caller's pinned revision.""" + if Path(model).is_dir(): + return Path(model) / filename + from huggingface_hub import hf_hub_download + + return Path(hf_hub_download(model, filename, revision=revision)) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + for name in ("target-model", "target-revision", "draft-model", "draft-revision"): + parser.add_argument(f"--{name}", required=True) + parser.add_argument("--tokens", required=True, type=int) + parser.add_argument("--thinking-mode", required=True) + parser.add_argument("--eval-only", required=True, choices=("true", "false")) + parser.add_argument("--run-eval", required=True, choices=("true", "false")) + parser.add_argument("--golden-file", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + args = parser.parse_args() + target = json.loads( + configuration_file(args.target_model, args.target_revision, "config.json").read_text() + ) + draft = json.loads( + configuration_file(args.draft_model, args.draft_revision, "config.json").read_text() + ) + draft_index = json.loads( + configuration_file( + args.draft_model, args.draft_revision, "model.safetensors.index.json" + ).read_text() + ) + spec, overrides = build_configs( + target, + draft, + draft_index["weight_map"], + yaml.safe_load(args.golden_file.read_text()), + draft_model=args.draft_model, + draft_revision=args.draft_revision, + tokens=args.tokens, + thinking_mode=args.thinking_mode, + eval_only=args.eval_only == "true", + run_eval=args.run_eval == "true", + ) + (args.output_dir / "speculative-config.json").write_text(json.dumps(spec)) + (args.output_dir / "hf-overrides.json").write_text(json.dumps(overrides)) + print(json.dumps(spec)) + + +if __name__ == "__main__": + main() diff --git a/infx/bench_serving/speedbench_acceptance.py b/infx/bench_serving/speedbench_acceptance.py new file mode 100644 index 0000000000..415a36e1d5 --- /dev/null +++ b/infx/bench_serving/speedbench_acceptance.py @@ -0,0 +1,104 @@ +"""Compute auditable SPEED-Bench acceptance from real vLLM counter deltas.""" + +from __future__ import annotations + +import argparse +import json +import math +import re +from copy import deepcopy +from pathlib import Path + + +def mtp_quantization_overrides(target: dict, draft_weights: dict[str, str]) -> dict: + """Preserve native MTP modules when vLLM inherits target FP8 configuration.""" + quantization = target.get("quantization_config") + if not quantization: + return {} + if quantization.get("quant_method") != "fp8": + raise ValueError("Only FP8 target quantization is supported by this collector") + modules = sorted( + name.removesuffix(".weight") + for name in draft_weights + if name.startswith("mtp.") and name.endswith(".weight") + ) + if not modules: + raise ValueError("The original draft checkpoint contains no MTP weights") + override = deepcopy(quantization) + key = "ignored_layers" if override.get("ignored_layers") else "modules_to_not_convert" + override[key] = list(dict.fromkeys([*override.get(key, []), *modules])) + return {"quantization_config": override} + + +def read_counters(text: str) -> dict[str, float]: + """Sum engine-labelled counters, rejecting missing or malformed measurements.""" + counters = {} + for key in ("num_drafts", "num_draft_tokens", "num_accepted_tokens"): + name = f"vllm:spec_decode_{key}_total" + values = [ + float(match[1]) + for match in re.finditer( + rf"^{name}(?:\{{[^\n]*\}})?\s+(\S+)(?:\s+\S+)?$", text, re.MULTILINE + ) + ] + if not values or any(not math.isfinite(v) or v < 0 for v in values): + raise ValueError(f"Missing or invalid counter: {name}") + counters[key] = sum(values) + return counters + + +def acceptance(before: str, after: str, draft_length: int) -> dict[str, float]: + """AL includes the bonus token; AR uses the actual proposed-token count.""" + start, end = read_counters(before), read_counters(after) + delta = {key: end[key] - start[key] for key in start} + drafts = delta["num_drafts"] + proposed = delta["num_draft_tokens"] + accepted = delta["num_accepted_tokens"] + if ( + draft_length < 1 + or drafts <= 0 + or proposed <= 0 + or accepted < 0 + or accepted > proposed + or proposed > draft_length * drafts + ): + raise ValueError(f"Invalid speculative counter deltas: {delta}") + return {**delta, "al": 1 + accepted / drafts, "ar": accepted / proposed} + + +def collect_cell( + before: str, after: str, result: dict, expected_prompts: int, draft_length: int +) -> dict[str, float]: + """Only publish a cell when all selected requests completed successfully.""" + if expected_prompts <= 0 or result.get("completed") != expected_prompts: + raise ValueError(f"Expected {expected_prompts} completions; got {result.get('completed')}") + errors = result.get("errors", []) + if any(errors): + raise ValueError("Benchmark contains request errors") + return acceptance(before, after, draft_length) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("before", type=Path) + parser.add_argument("after", type=Path) + parser.add_argument("result", type=Path) + parser.add_argument("expected_prompts", type=int) + parser.add_argument("draft_length", type=int) + args = parser.parse_args() + print( + json.dumps( + collect_cell( + args.before.read_text(), + args.after.read_text(), + json.loads(args.result.read_text()), + args.expected_prompts, + args.draft_length, + ), + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c5b4309f5f..f6b9ae3f07 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8455,3 +8455,31 @@ description: - "Update B200 vLLM AgentX to DSpark6 and a new image with TP8 and DEP8 configurations." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3274 + +- config-keys: + - qwen3.827b-fp8-h100-vllm-mtp + description: + - "Add the H100 Qwen3.8-27B-FP8 recipe: vLLM nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 on one GPU (TP1), 1k1k only, with the checkpoint's native MTP head (mtp_num_hidden_layers=1, --speculative-config method mtp) drafting three tokens as the vLLM recipe page prescribes, fp8 KV cache, --max-num-seqs sized to the sweep point (floor 16; 256 for accuracy evals), prefix caching disabled, the matrix context as --max-model-len, --language-model-only, and chat-templated prompts; concurrency 1-128" + - "新增 H100 Qwen3.8-27B-FP8 配方:vLLM nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 单卡(TP1),仅 1k1k,使用 checkpoint 自带的原生 MTP 头(mtp_num_hidden_layers=1,--speculative-config method mtp)按 vLLM recipe 页面建议预测 3 个 token,fp8 KV cache,--max-num-seqs 按并发点设定(下限 16;精度评测为 256),关闭 prefix caching,--max-model-len 使用矩阵上下文,--language-model-only,提示词经 chat template 处理;并发 1-128" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3292 + +- config-keys: + - qwen3.827b-fp8-h100-vllm-mtp + description: + - "Apply SPEED-Bench thinking-on golden AL 2.52 to Qwen3.8-27B-FP8 native MTP3 throughput on h100; preserve the original BF16 MTP head, computation and draft KV cache with pinned revisions and quantization exclusions; accuracy paths retain real verification" + - "在 h100 的 Qwen3.8-27B-FP8 原生 MTP3 吞吐测试中应用 SPEED-Bench thinking-on 黄金 AL 2.52;固定模型版本并排除草稿量化,保持原始 BF16 MTP 头、计算和草稿 KV cache;精度评测路径保留真实验证" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3292 + +- config-keys: + - qwen3.827b-fp8-h100-vllm-mtp + description: + - "Disable optional FlashInfer autotuning for Qwen3.8-27B-FP8 native BF16 MTP: the pinned CUDA image's dummy prefill fails FA3 scheduler_metadata shape validation at low concurrency; retain synthetic AL 2.52" + - "关闭 Qwen3.8-27B-FP8 原生 BF16 MTP 的可选 FlashInfer autotuning:固定 CUDA 镜像的 dummy prefill 在低并发时触发 FA3 scheduler_metadata 形状检查失败;保留合成 AL 2.52" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3292 + +- config-keys: + - qwen3.827b-fp8-h100-vllm-mtp + description: + - "Add 8192/1024 coverage on H100 at TP1, concurrency 1-128, alongside 1024/1024; native MTP3 throughput uses thinking-on golden AL 2.52 and accuracy evals retain real verification" + - "在 H100 上新增 8192/1024,使用 TP1、并发 1-128,并保留 1024/1024;原生 MTP3 吞吐测试使用 thinking-on 黄金 AL 2.52,精度评测保留真实验证" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3292 diff --git a/utils/test_qwen_mtp.py b/utils/test_qwen_mtp.py new file mode 100644 index 0000000000..0f931230c2 --- /dev/null +++ b/utils/test_qwen_mtp.py @@ -0,0 +1,56 @@ +import pytest + +from infx.bench_serving.qwen_mtp import build_configs + + +def prepare(*, mode="thinking_on", tokens=3, eval_only=False, run_eval=False, golden=None, draft=None): + return build_configs( + {"quantization_config": {"quant_method": "fp8", "modules_to_not_convert": ["lm_head"]}}, + draft if draft is not None else {"text_config": {"dtype": "bfloat16"}}, + {"mtp.fc.weight": "shard", "mtp.layers.0.mlp.down_proj.weight": "shard"}, + golden if golden is not None else { + "qwen3.8-27b-fp8": {"thinking_on": {3: 2.5}, "thinking_off": {3: 3.25}}, + }, + draft_model="reference/head", draft_revision="reference-revision", tokens=tokens, + thinking_mode=mode, eval_only=eval_only, run_eval=run_eval, + ) + + +@pytest.mark.parametrize("mode,expected", [("thinking_on", 2.5), ("thinking_off", 3.25)]) +def test_throughput_selects_measured_mode_and_preserves_native_head(mode, expected): + spec, overrides = prepare(mode=mode) + assert spec == { + "method": "mtp", "model": "reference/head", "revision": "reference-revision", + "num_speculative_tokens": 3, "kv_cache_dtype": "auto", + "rejection_sample_method": "synthetic", "synthetic_acceptance_length": expected, + } + assert overrides["quantization_config"]["modules_to_not_convert"] == [ + "lm_head", "mtp.fc", "mtp.layers.0.mlp.down_proj", + ] + + +@pytest.mark.parametrize("eval_only,run_eval", [(True, False), (False, True)]) +def test_accuracy_paths_keep_real_verification_without_a_golden_curve(eval_only, run_eval): + spec, _ = prepare(eval_only=eval_only, run_eval=run_eval, golden={}) + assert spec["rejection_sample_method"] == "standard" + assert "synthetic_acceptance_length" not in spec + + +@pytest.mark.parametrize("tokens", [0, 5]) +def test_unmeasured_draft_lengths_rejected(tokens): + with pytest.raises(ValueError, match="1-4 draft tokens"): + prepare(tokens=tokens) + + +@pytest.mark.parametrize("golden", [ + {}, {"qwen3.8-27b-fp8": {"thinking_on": {3: 4.1}}}, + {"qwen3.8-27b-fp8": {"thinking_on": {3: float("nan")}}}, +]) +def test_missing_or_invalid_acceptance_rejected(golden): + with pytest.raises(ValueError, match="golden AL"): + prepare(golden=golden) + + +def test_converted_draft_precision_rejected(): + with pytest.raises(ValueError, match="original BF16"): + prepare(draft={"text_config": {"dtype": "float16"}}) diff --git a/utils/test_speedbench_acceptance.py b/utils/test_speedbench_acceptance.py new file mode 100644 index 0000000000..44c29c864f --- /dev/null +++ b/utils/test_speedbench_acceptance.py @@ -0,0 +1,69 @@ +import pytest + +from infx.bench_serving.speedbench_acceptance import ( + acceptance, collect_cell, mtp_quantization_overrides, read_counters, +) + + +@pytest.mark.parametrize("key", ["modules_to_not_convert", "ignored_layers"]) +def test_native_mtp_exclusions_preserve_target_quantization(key): + target = {"quantization_config": {"quant_method": "fp8", key: ["lm_head"], "fmt": "e4m3"}} + result = mtp_quantization_overrides(target, { + "mtp.layers.0.mlp.down_proj.weight": "shard", "mtp.fc.weight": "shard", + "model.layers.0.mlp.down_proj.weight": "other", + }) + assert result == {"quantization_config": { + "quant_method": "fp8", "fmt": "e4m3", + key: ["lm_head", "mtp.fc", "mtp.layers.0.mlp.down_proj"], + }} + assert target["quantization_config"][key] == ["lm_head"] + + +def test_bf16_target_needs_no_quantization_override(): + assert mtp_quantization_overrides({"text_config": {"dtype": "bfloat16"}}, {}) == {} + + +def test_quantized_target_without_native_head_fails(): + with pytest.raises(ValueError, match="no MTP weights"): + mtp_quantization_overrides({"quantization_config": {"quant_method": "fp8"}}, {}) + + +def counters(drafts, proposed, accepted): + return ( + f'vllm:spec_decode_num_drafts_total{{engine="0"}} {drafts}\n' + f'vllm:spec_decode_num_draft_tokens_total{{engine="0"}} {proposed}\n' + f'vllm:spec_decode_num_accepted_tokens_total{{engine="0"}} {accepted}\n' + ) + + +def test_counter_deltas_use_actual_proposals_and_include_bonus(): + result = collect_cell(counters(10, 30, 20), counters(14, 40, 26), {"completed": 2}, 2, 3) + assert result == { + "num_drafts": 4, "num_draft_tokens": 10, "num_accepted_tokens": 6, + "al": 2.5, "ar": 0.6, + } + + +def test_sum_engines_and_parse_scientific_notation(): + text = counters("1e2", "3e2", "2e2") + counters(10, 30, 20).replace('"0"', '"1"') + assert read_counters(text) == { + "num_drafts": 110, "num_draft_tokens": 330, "num_accepted_tokens": 220, + } + + +@pytest.mark.parametrize("text", ["", counters(1, 3, "NaN"), counters(1, 3, -1)]) +def test_invalid_counters_fail(text): + with pytest.raises(ValueError, match="Missing or invalid counter"): + read_counters(text) + + +@pytest.mark.parametrize("after", [counters(0, 0, 0), counters(1, 4, 2), counters(1, 3, 4)]) +def test_invalid_deltas_fail(after): + with pytest.raises(ValueError, match="Invalid speculative counter deltas"): + acceptance(counters(0, 0, 0), after, 3) + + +@pytest.mark.parametrize("result", [{"completed": 1}, {"completed": 2, "errors": ["HTTP 500"]}]) +def test_incomplete_or_failed_benchmarks_fail(result): + with pytest.raises(ValueError): + collect_cell(counters(0, 0, 0), counters(4, 10, 6), result, 2, 3)