diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index de6aaf6b88..c5b6ea3791 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -1,4 +1,5 @@ name: SpeedBench AL Collection +run-name: "SPEED-Bench ${{ inputs.model }} | ${{ inputs.precision }} | k=[${{ inputs.mtp-list }}] | thinking=[${{ inputs.thinking-modes }}]" # Push-button (workflow_dispatch) collection of a SPEED-Bench acceptance-length # (AL) matrix: thinking_on/off x MTP levels, for the given model (defaults to @@ -30,6 +31,25 @@ on: # zizmor: ignore[concurrency-limits] required: false type: string default: 'vllm/vllm-openai:v0.21.0' + precision: + description: "Target precision" + type: string + default: 'fp4' + tp: + description: "Tensor parallel size and allocated GPU count" + type: string + default: '8' + collector-script: + description: "Optional model-specific collector path" + type: string + model-revision: + description: "Model checkpoint revision for revision-aware collectors" + type: string + default: 'main' + speculative-config: + description: "Real draft configuration; collector supplies num_speculative_tokens" + type: string + default: '{"method":"mtp"}' mtp-list: description: "Space-separated MTP levels (num_speculative_tokens)" required: false @@ -96,17 +116,25 @@ env: # the launcher resolves MODEL_PATH to the pre-staged local weights and mounts # them. The collector serves from MODEL_PATH (see SERVE_MODEL), so no download. MODEL: ${{ inputs.model }} + MODEL_PATH: /mnt/hf_hub_cache/speedbench-${{ inputs.model-prefix }}-${{ inputs.precision }} MODEL_PREFIX: ${{ inputs.model-prefix }} - PRECISION: fp4 + PRECISION: ${{ inputs.precision }} FRAMEWORK: vllm EXP_NAME: ${{ inputs.model-prefix }}_speedbench IMAGE: ${{ inputs.image }} - TP: '8' + TP: ${{ inputs.tp }} EP_SIZE: '1' DP_ATTENTION: 'false' SPEC_DECODING: mtp # Run the AL-matrix collector instead of the auto-selected throughput script. - BENCH_SCRIPT_OVERRIDE: benchmarks/single_node/speedbench/${{ inputs.model-prefix }}_fp4_b300_vllm.sh + BENCH_SCRIPT_OVERRIDE: ${{ inputs.collector-script || format('benchmarks/single_node/speedbench/{0}_fp4_b300_vllm.sh', inputs.model-prefix) }} + MODEL_REVISION: ${{ inputs.model-revision }} + SPEEDBENCH_SPECULATIVE_CONFIG: ${{ inputs.speculative-config }} + SPEEDBENCH_CONCURRENCY: '64' + SPEEDBENCH_MAX_MODEL_LEN: '16384' + SPEEDBENCH_GPU_MEMORY_UTILIZATION: '0.90' + SPEEDBENCH_SEED: '0' + SPEEDBENCH_PREPARE_REVISION: 'bcf059af55c20a89f797724598f9908d126153e6' SALLOC_TIME_LIMIT: ${{ inputs.salloc-time }} # Matrix-collector tunables (propagated into the container via srun --export=ALL). MTP_LIST: ${{ inputs.mtp-list }} @@ -133,15 +161,17 @@ jobs: env: RUNNER: ${{ inputs.runner }} MODEL_PREFIX: ${{ inputs.model-prefix }} + TARGET_PRECISION: ${{ inputs.precision }} run: | entry=$(jq -cn \ --arg runner "$RUNNER" \ --arg model_prefix "$MODEL_PREFIX" \ + --arg precision "$TARGET_PRECISION" \ '[{ runner: $runner, framework: "vllm", "model-prefix": $model_prefix, - precision: "fp4", + precision: $precision, "spec-decoding": "mtp" }]') scored=$(printf '%s' "$entry" | @@ -156,29 +186,16 @@ jobs: runs-on: >- ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && - ( - vars.NODE_SLOT_SCHEDULER_ENABLED == 'true' && - format( - '["self-hosted",{0},{1},{2},{3}]', - toJSON(inputs.runner), - '"nodes:1"', - toJSON(format( - 'ci-job-{0}-{1}', - needs.setup.outputs.priority, - needs.setup.outputs.queue-token - )), - toJSON(format('ci-attempt-{0}', github.run_attempt)) - ) || - format( - '["self-hosted",{0},{1},{2}]', - toJSON(inputs.runner), - toJSON(format( - 'ci-job-{0}-{1}', - needs.setup.outputs.priority, - needs.setup.outputs.queue-token - )), - toJSON(format('ci-attempt-{0}', github.run_attempt)) - ) + format( + '["self-hosted",{0},{1},{2},{3}]', + toJSON(inputs.runner), + '"nodes:1"', + toJSON(format( + 'ci-job-{0}-{1}', + needs.setup.outputs.priority, + needs.setup.outputs.queue-token + )), + toJSON(format('ci-attempt-{0}', github.run_attempt)) ) || format('[{0}]', toJSON(inputs.runner)) ) }} @@ -223,6 +240,7 @@ jobs: - name: Cleanup stale outputs (pre-run) run: | rm -f speedbench-reference-al.yaml || true + rm -f speedbench-evidence.tar.gz || true rm -f gpu_metrics.csv || true rm -rf speed_bench_data || true @@ -287,6 +305,14 @@ jobs: --base main \ --head "$BRANCH" + - name: Upload revision-aware collector evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: speedbench-evidence-${{ inputs.model-prefix }}-${{ inputs.precision }} + path: speedbench-evidence.tar.gz + if-no-files-found: ignore + - name: Upload server logs if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/benchmarks/single_node/speedbench/qwen3.827b_vllm.sh b/benchmarks/single_node/speedbench/qwen3.827b_vllm.sh new file mode 100644 index 0000000000..3fbc28c3b6 --- /dev/null +++ b/benchmarks/single_node/speedbench/qwen3.827b_vllm.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# Qwen3.8-27B native MTP SPEED-Bench acceptance; both targets keep the BF16 head. +set -eo pipefail +source "$(dirname "$0")/../../benchmark_lib.sh" +check_env_vars MODEL MODEL_PATH MODEL_REVISION PRECISION TP IMAGE PORT MTP_LIST \ + THINKING_MODES CATEGORY SPEEDBENCH_OUTPUT_LEN CHAT_TEMPLATE_KWARGS_ON OUT_YAML \ + SPEEDBENCH_SPECULATIVE_CONFIG SPEEDBENCH_CONCURRENCY SPEEDBENCH_MAX_MODEL_LEN \ + SPEEDBENCH_GPU_MEMORY_UTILIZATION SPEEDBENCH_SEED SPEEDBENCH_PREPARE_REVISION \ + INFMAX_CONTAINER_WORKSPACE + +for mtp in $MTP_LIST; do + if [[ ! "$mtp" =~ ^[1-4]$ ]]; then + echo "Qwen3.8-27B collection requires draft lengths 1-4; got $mtp" >&2 + exit 1 + fi +done + +# Runtime directories stay outside /workspace; only final files are staged there. +SPEEDBENCH_SCRATCH=$(mktemp -d /tmp/inferencex-speedbench.XXXXXX) +export SPEEDBENCH_SCRATCH +RESULTS_DIR="$SPEEDBENCH_SCRATCH/results" +DATA_DIR="$SPEEDBENCH_SCRATCH/data" +mkdir -p "$RESULTS_DIR" "$DATA_DIR" +SERVER_PID="" +cleanup_server() { + if [[ -n "$SERVER_PID" ]]; then + stop_background_process_groups 0 30 10 "$SERVER_PID" + wait "$SERVER_PID" 2>/dev/null || true + SERVER_PID="" + fi +} +finish() { + local rc=$? + trap - EXIT + cleanup_server || rc=1 + tar -czf "$INFMAX_CONTAINER_WORKSPACE/speedbench-evidence.tar.gz" -C "$SPEEDBENCH_SCRATCH" . || rc=1 + exit "$rc" +} +trap finish EXIT + +hf download "$MODEL" --revision "$MODEL_REVISION" --local-dir "$MODEL_PATH" +cp "$MODEL_PATH/config.json" "$RESULTS_DIR/target-config.json" +cp "$MODEL_PATH/model.safetensors.index.json" "$RESULTS_DIR/target-weight-index.json" +python3 - <<'PY' +import json, os +from pathlib import Path +from huggingface_hub import snapshot_download +from infx.bench_serving.speedbench_acceptance import mtp_quantization_overrides +spec = json.loads(os.environ['SPEEDBENCH_SPECULATIVE_CONFIG']) +if spec['method'] != 'mtp' or spec.get('rejection_sample_method') == 'synthetic': + raise ValueError('Collection requires real native MTP verification') +if spec['model'] != 'Qwen/Qwen3.8-27B': + raise ValueError('Use the original Qwen3.8-27B MTP head') +if spec.get('quantization') or spec.get('kv_cache_dtype') != 'auto': + raise ValueError('Keep the original BF16 draft weights, compute and KV cache') +draft = snapshot_download(spec['model'], revision=spec['revision']) +config = json.loads((Path(draft) / 'config.json').read_text()) +text = config.get('text_config', config) +if config.get('quantization_config') or text.get('dtype', text.get('torch_dtype')) != 'bfloat16': + raise ValueError('Draft must be the original unquantized BF16 checkpoint') +out = Path(os.environ['SPEEDBENCH_SCRATCH']) / 'results' +(out / 'draft-config.json').write_text(json.dumps(config, indent=2)) +draft_index = json.loads((Path(draft) / 'model.safetensors.index.json').read_text()) +target_config = json.loads((out / 'target-config.json').read_text()) +overrides = mtp_quantization_overrides(target_config, draft_index['weight_map']) +(out / 'hf-overrides.json').write_text(json.dumps(overrides, indent=2)) +metadata = {key: os.environ[key] for key in ( + 'MODEL', 'MODEL_REVISION', 'PRECISION', 'IMAGE', 'TP', 'MTP_LIST', 'THINKING_MODES', + 'CATEGORY', 'SPEEDBENCH_OUTPUT_LEN', 'CHAT_TEMPLATE_KWARGS_ON', + 'SPEEDBENCH_SPECULATIVE_CONFIG', 'SPEEDBENCH_CONCURRENCY', 'SPEEDBENCH_SEED', + 'SPEEDBENCH_PREPARE_REVISION')} +metadata['draft_snapshot'] = Path(draft).name +(out / 'metadata.json').write_text(json.dumps(metadata, indent=2)) +PY +pip install -q datasets tiktoken pyyaml +curl -LsSf "https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/${SPEEDBENCH_PREPARE_REVISION}/nemo_skills/dataset/speed-bench/prepare.py" \ + -o "$SPEEDBENCH_SCRATCH/prepare.py" +python3 "$SPEEDBENCH_SCRATCH/prepare.py" --config qualitative --output_dir "$DATA_DIR" +EXPECTED_PROMPTS=$(python3 - "$DATA_DIR/qualitative.jsonl" "$CATEGORY" <<'PY' +import json, sys +with open(sys.argv[1]) as stream: + count = sum(json.loads(line)['category'] == sys.argv[2] for line in stream) +if count <= 0: + raise ValueError('No prompts in requested category') +print(count) +PY +) +sha256sum "$DATA_DIR/qualitative.jsonl" > "$RESULTS_DIR/dataset.sha256" +vllm --version > "$RESULTS_DIR/vllm-version.txt" +nvidia-smi > "$RESULTS_DIR/nvidia-smi.txt" + +for mode in $THINKING_MODES; do + case "$mode" in + on) temp=1.0; top_p=0.95; penalty=0.0; kwargs="$CHAT_TEMPLATE_KWARGS_ON" ;; + off) temp=0.7; top_p=0.8; penalty=1.5; kwargs='{"enable_thinking":false}' ;; + *) echo "Invalid thinking mode: $mode" >&2; exit 1 ;; + esac + for mtp in $MTP_LIST; do + SPEC_CONFIG=$(python3 - "$SPEEDBENCH_SPECULATIVE_CONFIG" "$mtp" <<'PY' +import json, sys +config = json.loads(sys.argv[1]) +config['num_speculative_tokens'] = int(sys.argv[2]) +print(json.dumps(config)) +PY +) + select_available_server_port + CELL="$RESULTS_DIR/${mode}_${mtp}" + mkdir -p "$CELL" + SERVER_LOG="$CELL/server.log" + SERVER_COMMAND=(vllm serve "$MODEL_PATH" --served-model-name "$MODEL" + --host 0.0.0.0 --port "$PORT" --tensor-parallel-size "$TP" + --dtype bfloat16 --language-model-only --trust-remote-code + --kv-cache-dtype fp8 --no-enable-prefix-caching + --max-model-len "$SPEEDBENCH_MAX_MODEL_LEN" + --max-num-seqs "$SPEEDBENCH_CONCURRENCY" + --gpu-memory-utilization "$SPEEDBENCH_GPU_MEMORY_UTILIZATION" + --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice + --seed "$SPEEDBENCH_SEED" --speculative-config "$SPEC_CONFIG" + --disable-uvicorn-access-log) + if [[ "$PRECISION" == "fp8" ]]; then + SERVER_COMMAND+=(--hf-overrides "$(cat "$RESULTS_DIR/hf-overrides.json")") + fi + printf '%q ' "${SERVER_COMMAND[@]}" > "$CELL/server-command.txt" + printf '\n' >> "$CELL/server-command.txt" + echo "Starting thinking=$mode draft_length=$mtp precision=$PRECISION" + VLLM_LOG_MODEL_INSPECTION=1 setsid "${SERVER_COMMAND[@]}" > "$SERVER_LOG" 2>&1 & + SERVER_PID=$! + wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + curl -fsS "http://localhost:$PORT/metrics" > "$CELL/before.prom" + BENCH_COMMAND=(vllm bench serve --model "$MODEL_PATH" --served-model-name "$MODEL" + --port "$PORT" --dataset-name speed_bench --dataset-path "$DATA_DIR" + --speed-bench-category "$CATEGORY" --speed-bench-output-len "$SPEEDBENCH_OUTPUT_LEN" + --num-prompts -1 --max-concurrency "$SPEEDBENCH_CONCURRENCY" + --save-result --save-detailed --result-dir "$CELL" --result-filename benchmark.json + --trust-remote-code --temperature "$temp" --top-p "$top_p" --top-k 20 + --presence-penalty "$penalty" --seed "$SPEEDBENCH_SEED" + --num-warmups 0 --ready-check-timeout-sec 0 --chat-template-kwargs "$kwargs") + printf '%q ' "${BENCH_COMMAND[@]}" > "$CELL/benchmark-command.txt" + printf '\n' >> "$CELL/benchmark-command.txt" + "${BENCH_COMMAND[@]}" 2>&1 | tee "$CELL/benchmark.log" + # vLLM exports asynchronously; allow its final stats interval to flush. + sleep 10 + curl -fsS "http://localhost:$PORT/metrics" > "$CELL/after.prom" + python3 -m infx.bench_serving.speedbench_acceptance \ + "$CELL/before.prom" "$CELL/after.prom" "$CELL/benchmark.json" "$EXPECTED_PROMPTS" "$mtp" \ + | tee "$CELL/acceptance.json" + cleanup_server + done +done + +python3 - <<'PY' +import json, os +from pathlib import Path +import yaml +root = Path(os.environ['SPEEDBENCH_SCRATCH']) / 'results' +model = os.environ['MODEL'].split('/')[-1].lower() +al, ar = {}, {} +for mode in os.environ['THINKING_MODES'].split(): + al[f'thinking_{mode}'], ar[f'thinking_{mode}'] = {}, {} + for length in map(int, os.environ['MTP_LIST'].split()): + result = json.loads((root / f'{mode}_{length}' / 'acceptance.json').read_text()) + al[f'thinking_{mode}'][length] = round(result['al'], 2) + ar[f'thinking_{mode}'][length] = result['ar'] +header = '\n'.join('# ' + line for line in (root / 'metadata.json').read_text().splitlines()) + '\n' +Path(os.environ['OUT_YAML']).write_text(header + yaml.safe_dump({model: al}, sort_keys=False)) +(root / 'acceptance-rates.yaml').write_text(yaml.safe_dump({model: ar}, sort_keys=False)) +print(Path(os.environ['OUT_YAML']).read_text()) +PY diff --git a/golden_al_distribution/README.md b/golden_al_distribution/README.md index 061851eee9..3eacc702d0 100644 --- a/golden_al_distribution/README.md +++ b/golden_al_distribution/README.md @@ -121,6 +121,44 @@ Before accepting an updated curve, reviewers should verify: - The source Actions run is linked at the first line of the YAML. - The committed values exactly match the workflow artifact. +## Qwen3.8-27B collection + +The revision-aware collector `benchmarks/single_node/speedbench/qwen3.827b_vllm.sh` +supports FP8 and BF16 targets with native MTP. Dispatch `speedbench-al.yml` +on `runner=b300` or `runner=cluster:h200-dgxc` (the H200 pool also accepts the +explicit collector override), +with `collector-script` set to that path, `precision`, `tp=1`, a pinned +`model-revision`, and an explicit `speculative-config` (the collector supplies +`num_speculative_tokens`). Pass `mtp-list=1 2 3 4`; this collector rejects draft +lengths outside 1–4 before downloading models or starting a server. Use the Qwen chat setting +`thinking-kwargs={"enable_thinking":true}`. Sampling follows the model card: +thinking on uses temperature 1.0, top-p 0.95 and presence penalty 0; thinking off +uses 0.7, 0.8 and 1.5, respectively; both use top-k 20. + +The official FP8 checkpoint also quantizes its embedded MTP head. To preserve +the original draft precision, native MTP collection must explicitly select +`model=Qwen/Qwen3.8-27B` and its pinned BF16 revision inside `speculative-config`, +with `kv_cache_dtype=auto`. Both target precisions use this same original MTP head. +The pinned vLLM MTP loader also inherits the target quantization configuration. +For FP8, the collector adds every original `mtp.*` weight module to that +configuration's exclusion list through `--hf-overrides`, preserving all target +settings and existing exclusions. Without these exclusions, selecting the BF16 +draft checkpoint alone can cast its weights into FP8 modules with invalid scales. +The evidence includes the exact overrides and vLLM model inspection output, so +the MTP linear modules can be verified as unquantized before accepting a curve. +These measurements do not validate recipes that quantize +the draft head or inherit the target's FP8 KV cache for the draft. + +All selected coding prompts must complete before a cell is accepted. The collector +disables the benchmark client's extra readiness request and warmups, and retains +before/after Prometheus counters. AL is `1 + accepted / drafts`; AR is +`accepted / proposed_tokens`, using actual proposals rather than an assumed +fixed draft length. `speedbench-evidence--` contains +commands, checkpoint metadata, the prepared dataset, per-request outputs, logs, +unrounded counters/AL/AR, and an AR matrix. Runtime directories are created under +`/tmp`; the final YAML and evidence archive are staged as workspace-root files. +Failed or incomplete cells never produce a golden matrix. + ## Current golden curves | Model | Method | Golden YAML | Source run | @@ -136,6 +174,11 @@ Before accepting an updated curve, reviewers should verify: | MiniMax-M3 | EAGLE3 (GQA) | [`minimaxm3_eagle3_gqa.yaml`](minimaxm3_eagle3_gqa.yaml) | [29784780049](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/29784780049) | | GLM-5.2 | MTP | [`glm5.2_mtp.yaml`](glm5.2_mtp.yaml) | [28058352479](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28058352479) | | Qwen3.8-Flash-Next | MTP (native) | [`qwen3.8next_mtp.yaml`](qwen3.8next_mtp.yaml) | [33034290269](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/33034290269) | +| Qwen3.8-27B BF16 | Native MTP (original BF16 head) | [`qwen3.827b_bf16_mtp.yaml`](qwen3.827b_bf16_mtp.yaml) | [thinking off](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35492788796), [thinking on](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35492789880) | +| Qwen3.8-27B FP8 | Native MTP (original BF16 head) | [`qwen3.827b_fp8_mtp.yaml`](qwen3.827b_fp8_mtp.yaml) | [thinking off](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35492786441), [thinking on](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35492787451) | + +Both Qwen3.8-27B curves cover draft lengths 1–4 with thinking off/on. Every cell completed all 80 coding prompts with zero failures. The YAML comments retain per-point AR fractions and raw counters; the values and model inspection were checked against the workflow artifacts. + ## Primary references diff --git a/golden_al_distribution/README_zh.md b/golden_al_distribution/README_zh.md index 79474a4787..15b2067f7b 100644 --- a/golden_al_distribution/README_zh.md +++ b/golden_al_distribution/README_zh.md @@ -121,6 +121,40 @@ gh workflow run speedbench-al.yml \ - YAML 第一行链接了源 Actions run。 - 提交的数值与工作流 artifact 完全一致。 +## Qwen3.8-27B 收集 + +支持固定版本的收集器 `benchmarks/single_node/speedbench/qwen3.827b_vllm.sh` +可测量 FP8 和 BF16 目标模型的原生 MTP。触发 `speedbench-al.yml` +时,可选择 `runner=b300` 或 `runner=cluster:h200-dgxc`(H200 池同样支持显式 +指定收集器)。将 `collector-script` 设为该路径,并显式传入 `precision`、`tp=1`、固定的 +`model-revision` 和 `speculative-config`(由收集器逐点填入 +`num_speculative_tokens`)。传入 `mtp-list=1 2 3 4`;收集器会在下载模型或启动服务前 +拒绝超出 1–4 范围的草稿长度。Qwen 使用 +`thinking-kwargs={"enable_thinking":true}`。采样遵循模型卡:thinking 开启时, +temperature 为 1.0、top-p 为 0.95、presence penalty 为 0;关闭时分别为 +0.7、0.8 和 1.5;两种模式的 top-k 均为 20。 + +官方 FP8 checkpoint 同时量化了内嵌 MTP 头。为保持原始草稿精度,收集原生 MTP +时必须在 `speculative-config` 中显式指定 `model=Qwen/Qwen3.8-27B` 及其固定的 +BF16 revision,并设置 `kv_cache_dtype=auto`。两种目标精度均使用同一原始 MTP 头。 +固定版本的 vLLM MTP loader 还会继承目标模型的量化配置。因此,收集 FP8 时, +收集器通过 `--hf-overrides` 将原始 `mtp.*` 权重对应的所有模块加入量化排除列表, +并保留目标模型的其他设置和已有排除项。如果缺少这些排除项,仅指定 BF16 草稿 +checkpoint 仍可能把权重转换到 FP8 模块中,同时使用无效的缩放因子。 +证据中包含完整覆盖配置和 vLLM 模型检查输出,接受曲线前可以据此确认 MTP +线性模块未量化。 +这些测量不能 +作为量化草稿头或让草稿继承目标模型 FP8 KV cache 的配方的有效性证明。 + +只有所选 coding 提示词全部成功完成,测量点才会被接受。收集器关闭基准客户端 +额外的就绪检查请求和预热,并保留运行前后的 Prometheus 计数器。 +AL 为 `1 + accepted / drafts`;AR 为 `accepted / proposed_tokens`,使用实际 +提出的草稿 token 数,而非假设每轮草稿长度固定。 +`speedbench-evidence--` 包含命令、checkpoint 元数据、 +处理后的数据集、逐请求输出、日志、未舍入的计数器与 AL/AR,以及 AR 矩阵。 +运行时目录位于 `/tmp`,最终 YAML 和证据归档以文件形式放在工作区根目录。 +失败或未完成的测量点不会生成黄金矩阵。 + ## 当前黄金曲线 | 模型 | 方法 | 黄金 YAML | 源 run | @@ -136,6 +170,11 @@ gh workflow run speedbench-al.yml \ | MiniMax-M3 | EAGLE3(GQA) | [`minimaxm3_eagle3_gqa.yaml`](minimaxm3_eagle3_gqa.yaml) | [29784780049](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/29784780049) | | GLM-5.2 | MTP | [`glm5.2_mtp.yaml`](glm5.2_mtp.yaml) | [28058352479](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28058352479) | | Qwen3.8-Flash-Next | MTP (native) | [`qwen3.8next_mtp.yaml`](qwen3.8next_mtp.yaml) | [33034290269](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/33034290269) | +| Qwen3.8-27B BF16 | 原生 MTP(原始 BF16 头) | [`qwen3.827b_bf16_mtp.yaml`](qwen3.827b_bf16_mtp.yaml) | [thinking 关闭](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35492788796)、[thinking 开启](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35492789880) | +| Qwen3.8-27B FP8 | 原生 MTP(原始 BF16 头) | [`qwen3.827b_fp8_mtp.yaml`](qwen3.827b_fp8_mtp.yaml) | [thinking 关闭](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35492786441)、[thinking 开启](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35492787451) | + +Qwen3.8-27B 的两条曲线均覆盖 1–4 个草稿 token,以及 thinking 关闭/开启两种模式。每个测量点均完成全部 80 个 coding 提示词,失败数为零。YAML 注释保留逐点 AR 比例和原始计数器;数值和模型检查结果已与工作流 artifact 核对。 + ## 主要参考资料 diff --git a/golden_al_distribution/qwen3.827b_bf16_mtp.yaml b/golden_al_distribution/qwen3.827b_bf16_mtp.yaml new file mode 100644 index 0000000000..3e37c93b05 --- /dev/null +++ b/golden_al_distribution/qwen3.827b_bf16_mtp.yaml @@ -0,0 +1,23 @@ +# Source GitHub Actions runs: thinking_off https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35492788796; thinking_on https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35492789880 +# SPEED-Bench Qualitative coding; all 80 requests succeeded in every cell; maximum output length 4096. +# Prepared dataset SHA256: e9bbbe998472b79501ed79ba2bf126e58b63e153a6deef70ce2b63ea7a4de845 +# Target: Qwen/Qwen3.8-27B @ 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0; target weights: bf16; target KV cache: fp8. +# Image: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3; H200; TP=1; concurrency=64; context=16384; seed=0. +# Native MTP head: Qwen/Qwen3.8-27B @ 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0; original BF16 weights, compute and KV cache. +# FP8 collection excludes all original MTP weight modules from inherited target quantization through --hf-overrides. +# thinking_on: temperature=1.0, top_p=0.95, top_k=20, presence_penalty=0.0, chat_template_kwargs={"enable_thinking":true}. +# thinking_off: temperature=0.7, top_p=0.8, top_k=20, presence_penalty=1.5, chat_template_kwargs={"enable_thinking":false}. +# AL includes the bonus token and is rounded to two decimals. AR comments are fractions, using actual proposed tokens. +# Counter comments retain accepted draft tokens, proposed draft tokens, and verification drafts for exact recomputation. +# Collector: benchmarks/single_node/speedbench/qwen3.827b_vllm.sh (speedbench-al.yml); draft lengths limited to 1-4. +qwen3.8-27b: + thinking_off: + 1: 1.91 # AR=0.911074663054; accepted=25552; proposed=28046; drafts=28046 + 2: 2.66 # AR=0.829343758995; accepted=34577; proposed=41692; drafts=20846 + 3: 3.27 # AR=0.755599710983; accepted=37647; proposed=49824; drafts=16608 + 4: 3.78 # AR=0.694046325228; accepted=40871; proposed=58888; drafts=14722 + thinking_on: + 1: 1.73 # AR=0.732153180956; accepted=89915; proposed=122809; drafts=122809 + 2: 2.21 # AR=0.604088456726; accepted=117790; proposed=194988; drafts=97494 + 3: 2.51 # AR=0.502475973156; accepted=126838; proposed=252426; drafts=84142 + 4: 2.72 # AR=0.428857766600; accepted=134368; proposed=313316; drafts=78329 diff --git a/golden_al_distribution/qwen3.827b_fp8_mtp.yaml b/golden_al_distribution/qwen3.827b_fp8_mtp.yaml new file mode 100644 index 0000000000..b62e8d0e04 --- /dev/null +++ b/golden_al_distribution/qwen3.827b_fp8_mtp.yaml @@ -0,0 +1,23 @@ +# Source GitHub Actions runs: thinking_off https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35492786441; thinking_on https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35492787451 +# SPEED-Bench Qualitative coding; all 80 requests succeeded in every cell; maximum output length 4096. +# Prepared dataset SHA256: e9bbbe998472b79501ed79ba2bf126e58b63e153a6deef70ce2b63ea7a4de845 +# Target: Qwen/Qwen3.8-27B-FP8 @ 017b9c7af6b5689d5dd426a76e0bc077eb5ca20a; target weights: fp8; target KV cache: fp8. +# Image: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3; H200; TP=1; concurrency=64; context=16384; seed=0. +# Native MTP head: Qwen/Qwen3.8-27B @ 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0; original BF16 weights, compute and KV cache. +# FP8 collection excludes all original MTP weight modules from inherited target quantization through --hf-overrides. +# thinking_on: temperature=1.0, top_p=0.95, top_k=20, presence_penalty=0.0, chat_template_kwargs={"enable_thinking":true}. +# thinking_off: temperature=0.7, top_p=0.8, top_k=20, presence_penalty=1.5, chat_template_kwargs={"enable_thinking":false}. +# AL includes the bonus token and is rounded to two decimals. AR comments are fractions, using actual proposed tokens. +# Counter comments retain accepted draft tokens, proposed draft tokens, and verification drafts for exact recomputation. +# Collector: benchmarks/single_node/speedbench/qwen3.827b_vllm.sh (speedbench-al.yml); draft lengths limited to 1-4. +qwen3.8-27b-fp8: + thinking_off: + 1: 1.90 # AR=0.900382608696; accepted=25886; proposed=28750; drafts=28750 + 2: 2.68 # AR=0.837843967342; accepted=33249; proposed=39684; drafts=19842 + 3: 3.28 # AR=0.759643860940; accepted=40271; proposed=53013; drafts=17671 + 4: 3.79 # AR=0.698515476943; accepted=39807; proposed=56988; drafts=14247 + thinking_on: + 1: 1.73 # AR=0.726705818716; accepted=94043; proposed=129410; drafts=129410 + 2: 2.21 # AR=0.604951065637; accepted=119669; proposed=197816; drafts=98908 + 3: 2.52 # AR=0.507043631751; accepted=134722; proposed=265701; drafts=88567 + 4: 2.72 # AR=0.430686744384; accepted=135513; proposed=314644; drafts=78661 diff --git a/infx/bench_serving/speedbench_acceptance.py b/infx/bench_serving/speedbench_acceptance.py new file mode 100644 index 0000000000..415a36e1d5 --- /dev/null +++ b/infx/bench_serving/speedbench_acceptance.py @@ -0,0 +1,104 @@ +"""Compute auditable SPEED-Bench acceptance from real vLLM counter deltas.""" + +from __future__ import annotations + +import argparse +import json +import math +import re +from copy import deepcopy +from pathlib import Path + + +def mtp_quantization_overrides(target: dict, draft_weights: dict[str, str]) -> dict: + """Preserve native MTP modules when vLLM inherits target FP8 configuration.""" + quantization = target.get("quantization_config") + if not quantization: + return {} + if quantization.get("quant_method") != "fp8": + raise ValueError("Only FP8 target quantization is supported by this collector") + modules = sorted( + name.removesuffix(".weight") + for name in draft_weights + if name.startswith("mtp.") and name.endswith(".weight") + ) + if not modules: + raise ValueError("The original draft checkpoint contains no MTP weights") + override = deepcopy(quantization) + key = "ignored_layers" if override.get("ignored_layers") else "modules_to_not_convert" + override[key] = list(dict.fromkeys([*override.get(key, []), *modules])) + return {"quantization_config": override} + + +def read_counters(text: str) -> dict[str, float]: + """Sum engine-labelled counters, rejecting missing or malformed measurements.""" + counters = {} + for key in ("num_drafts", "num_draft_tokens", "num_accepted_tokens"): + name = f"vllm:spec_decode_{key}_total" + values = [ + float(match[1]) + for match in re.finditer( + rf"^{name}(?:\{{[^\n]*\}})?\s+(\S+)(?:\s+\S+)?$", text, re.MULTILINE + ) + ] + if not values or any(not math.isfinite(v) or v < 0 for v in values): + raise ValueError(f"Missing or invalid counter: {name}") + counters[key] = sum(values) + return counters + + +def acceptance(before: str, after: str, draft_length: int) -> dict[str, float]: + """AL includes the bonus token; AR uses the actual proposed-token count.""" + start, end = read_counters(before), read_counters(after) + delta = {key: end[key] - start[key] for key in start} + drafts = delta["num_drafts"] + proposed = delta["num_draft_tokens"] + accepted = delta["num_accepted_tokens"] + if ( + draft_length < 1 + or drafts <= 0 + or proposed <= 0 + or accepted < 0 + or accepted > proposed + or proposed > draft_length * drafts + ): + raise ValueError(f"Invalid speculative counter deltas: {delta}") + return {**delta, "al": 1 + accepted / drafts, "ar": accepted / proposed} + + +def collect_cell( + before: str, after: str, result: dict, expected_prompts: int, draft_length: int +) -> dict[str, float]: + """Only publish a cell when all selected requests completed successfully.""" + if expected_prompts <= 0 or result.get("completed") != expected_prompts: + raise ValueError(f"Expected {expected_prompts} completions; got {result.get('completed')}") + errors = result.get("errors", []) + if any(errors): + raise ValueError("Benchmark contains request errors") + return acceptance(before, after, draft_length) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("before", type=Path) + parser.add_argument("after", type=Path) + parser.add_argument("result", type=Path) + parser.add_argument("expected_prompts", type=int) + parser.add_argument("draft_length", type=int) + args = parser.parse_args() + print( + json.dumps( + collect_cell( + args.before.read_text(), + args.after.read_text(), + json.loads(args.result.read_text()), + args.expected_prompts, + args.draft_length, + ), + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 36405594c2..115f860d9e 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -467,6 +467,10 @@ else LEGACY_FW_SUFFIX=$([[ "$FRAMEWORK" == "trt" ]] && printf '_trt' || printf '') BENCH_SCRIPT="${BENCH_BASE}${LEGACY_FW_SUFFIX}${SPEC_SUFFIX}.sh" fi + # Explicit collectors use the same pool launcher and allocation as benchmarks. + if [[ -n "${BENCH_SCRIPT_OVERRIDE:-}" ]]; then + BENCH_SCRIPT="$BENCH_SCRIPT_OVERRIDE" + fi # DeepSeek-V4.1-Flash creates AgentX runtime directories next to the # repository, which must not land under /workspace. diff --git a/utils/test_speedbench_acceptance.py b/utils/test_speedbench_acceptance.py new file mode 100644 index 0000000000..44c29c864f --- /dev/null +++ b/utils/test_speedbench_acceptance.py @@ -0,0 +1,69 @@ +import pytest + +from infx.bench_serving.speedbench_acceptance import ( + acceptance, collect_cell, mtp_quantization_overrides, read_counters, +) + + +@pytest.mark.parametrize("key", ["modules_to_not_convert", "ignored_layers"]) +def test_native_mtp_exclusions_preserve_target_quantization(key): + target = {"quantization_config": {"quant_method": "fp8", key: ["lm_head"], "fmt": "e4m3"}} + result = mtp_quantization_overrides(target, { + "mtp.layers.0.mlp.down_proj.weight": "shard", "mtp.fc.weight": "shard", + "model.layers.0.mlp.down_proj.weight": "other", + }) + assert result == {"quantization_config": { + "quant_method": "fp8", "fmt": "e4m3", + key: ["lm_head", "mtp.fc", "mtp.layers.0.mlp.down_proj"], + }} + assert target["quantization_config"][key] == ["lm_head"] + + +def test_bf16_target_needs_no_quantization_override(): + assert mtp_quantization_overrides({"text_config": {"dtype": "bfloat16"}}, {}) == {} + + +def test_quantized_target_without_native_head_fails(): + with pytest.raises(ValueError, match="no MTP weights"): + mtp_quantization_overrides({"quantization_config": {"quant_method": "fp8"}}, {}) + + +def counters(drafts, proposed, accepted): + return ( + f'vllm:spec_decode_num_drafts_total{{engine="0"}} {drafts}\n' + f'vllm:spec_decode_num_draft_tokens_total{{engine="0"}} {proposed}\n' + f'vllm:spec_decode_num_accepted_tokens_total{{engine="0"}} {accepted}\n' + ) + + +def test_counter_deltas_use_actual_proposals_and_include_bonus(): + result = collect_cell(counters(10, 30, 20), counters(14, 40, 26), {"completed": 2}, 2, 3) + assert result == { + "num_drafts": 4, "num_draft_tokens": 10, "num_accepted_tokens": 6, + "al": 2.5, "ar": 0.6, + } + + +def test_sum_engines_and_parse_scientific_notation(): + text = counters("1e2", "3e2", "2e2") + counters(10, 30, 20).replace('"0"', '"1"') + assert read_counters(text) == { + "num_drafts": 110, "num_draft_tokens": 330, "num_accepted_tokens": 220, + } + + +@pytest.mark.parametrize("text", ["", counters(1, 3, "NaN"), counters(1, 3, -1)]) +def test_invalid_counters_fail(text): + with pytest.raises(ValueError, match="Missing or invalid counter"): + read_counters(text) + + +@pytest.mark.parametrize("after", [counters(0, 0, 0), counters(1, 4, 2), counters(1, 3, 4)]) +def test_invalid_deltas_fail(after): + with pytest.raises(ValueError, match="Invalid speculative counter deltas"): + acceptance(counters(0, 0, 0), after, 3) + + +@pytest.mark.parametrize("result", [{"completed": 1}, {"completed": 2, "errors": ["HTTP 500"]}]) +def test_incomplete_or_failed_benchmarks_fail(result): + with pytest.raises(ValueError): + collect_cell(counters(0, 0, 0), counters(4, 10, 6), result, 2, 3)