diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/8k1k.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/8k1k.yaml index 98f877110c..e5c7437481 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/8k1k.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/8k1k.yaml @@ -8,6 +8,8 @@ base: resources: gpu_type: b300 gpus_per_node: 8 + srun_options: + mpi: pmix frontend: type: sglang enable_multiple_frontends: false @@ -43,11 +45,12 @@ base: stream-interval: 50 scheduler-recv-interval: 10 tokenizer-worker-num: 6 - tokenizer-path: Qwen/Qwen3.5-397B-A17B-FP8 + tokenizer-path: /model context-length: 9236 enable-metrics: false env: PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: '0' benchmark: type: custom command: bash /infmax-workspace/benchmarks/single_node/srt_fixed_sequence.sh @@ -56,13 +59,13 @@ base: ISL: '8192' OSL: '1024' RANDOM_RANGE_RATIO: '0.8' - USE_CHAT_TEMPLATE: 'false' + USE_CHAT_TEMPLATE: 'true' zip_override_concurrency: roles: agg: args: - cuda-graph-max-bs: [4, 8, 16, 32, 64, 128, 256] - max-running-requests: [4, 8, 16, 32, 64, 128, 256] + cuda-graph-max-bs: [1, 2, 4, 8, 16, 32, 64, 128, 256] + max-running-requests: [1, 2, 4, 8, 16, 32, 64, 128, 256] benchmark: env: - CONC: ['4', '8', '16', '32', '64', '128', '256'] + CONC: ['1', '2', '4', '8', '16', '32', '64', '128', '256'] diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index e38ba4bc9e..c1fd93e523 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1309,10 +1309,11 @@ qwen3.5-fp8-b300-sglang: fixed-seq-len: - isl: 8192 osl: 1024 + require-power: true search-space: - tp: 4 ep: 1 - conc-start: 4 + conc-start: 1 conc-end: 256 srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/8k1k.yaml diff --git a/infx/srt_slurm/single_node.py b/infx/srt_slurm/single_node.py index 31655f6dc7..5ab3c034ab 100644 --- a/infx/srt_slurm/single_node.py +++ b/infx/srt_slurm/single_node.py @@ -69,6 +69,17 @@ def select_recipe(config: str, environment: Mapping[str, str]) -> tuple[str, dic return matches[0] +def _qwen35_b300_fp8(recipe: dict[str, Any], environment: Mapping[str, str]) -> bool: + """Preserve the established Qwen B300 workload and diagnostic settings.""" + return ( + recipe["model"]["path"] == "hf:Qwen/Qwen3.5-397B-A17B-FP8" + and recipe["model"]["precision"] == "fp8" + and recipe["resources"].get("gpu_type") == "b300" + and environment["FRAMEWORK"] == "sglang" + and environment["SPEC_DECODING"] == "none" + ) + + def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> None: """Reject metadata mismatches without overwriting recipe-owned server settings.""" role = recipe["roles"]["agg"] @@ -96,7 +107,10 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N "benchmark type": (benchmark["type"], "custom"), "benchmark MODEL": (workload["MODEL"], environment["MODEL"]), "SPEC_DECODING": (speculation, environment["SPEC_DECODING"]), - "USE_CHAT_TEMPLATE": (workload["USE_CHAT_TEMPLATE"], "true" if spec else "false"), + "USE_CHAT_TEMPLATE": ( + workload["USE_CHAT_TEMPLATE"], + "true" if spec or _qwen35_b300_fp8(recipe, environment) else "false", + ), } if "CONC" in workload: expected["CONC"] = (str(workload["CONC"]), environment["CONC"]) @@ -168,6 +182,17 @@ def runtime_arguments(config: str, environment: Mapping[str, str]) -> list[str]: }[environment["FRAMEWORK"]] for key in context_keys: overrides += ["--set", f"roles.agg.args.{key}={context}"] + if _qwen35_b300_fp8(recipe, environment) and environment["CONC"] == "256": + overrides += [ + "--set", + "roles.agg.args.log-requests=true", + "--set", + "roles.agg.args.log-requests-level=3", + "--set", + 'roles.agg.args.log-requests-format="json"', + "--set", + 'roles.agg.env.SGLANG_LOG_REQUEST_EXCEEDED_MS="0"', + ] return [*overrides, "--set", 'benchmark.env.RESULT_DIR="/logs"'] diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1bc132cd37..f4377db6a7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8950,3 +8950,45 @@ description: - "Update SGLang image from v0.5.19-cu130 to v0.5.20-cu130 and pass --cuda-graph-max-bs-decode after the alias removal." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3334 + +- config-keys: + - qwen3.5-fp8-b300-sglang + scenario-type: + - fixed-seq-len + description: + - Preserve Qwen3.5 FP8 b300 coverage and require validated power; include the article + ladder where missing. + - 保留现有覆盖并要求通过验证的功耗，补齐缺失的文章并发梯度。 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3379 + +- config-keys: + - qwen3.5-fp8-b300-sglang + scenario-type: + - fixed-seq-len + description: + - Route Qwen3.5 FP8 STP through the existing B300 batch allocation path, retaining + its serving settings and measurements; the batch container step uses PMIx. + - 将 Qwen3.5 FP8 STP 接入现有 B300 batch 分配路径，保留服务参数和测量设置；batch 容器步骤使用 PMIx。 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3379 + +- config-keys: + - qwen3.5-fp8-b300-sglang + scenario-type: + - fixed-seq-len + description: + - Disable JIT DeepGEMM for Qwen3.5 FP8 B300 to validate the native Blackwell + scale-format accuracy warning with the existing FlashInfer fallback. + - 在 Qwen3.5 FP8 B300 上关闭 JIT DeepGEMM，以现有 FlashInfer 回退路径验证 Blackwell + scale-format 精度警告；保留模型、评测输入与准确率门槛。 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3379 + +- config-keys: + - qwen3.5-fp8-b300-sglang + scenario-type: + - fixed-seq-len + description: + - Preserve the validated DeepGEMM-off Qwen B300 recipe, chat-template workload, + PMIx launch and c1-c256 power coverage through the native SRT launcher. + - 在原生 SRT 启动路径中保留 Qwen B300 关闭 DeepGEMM 的配方、聊天模板、 + PMIx 和 c1-c256 功耗覆盖。 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3379 diff --git a/utils/test_srt_single_node.py b/utils/test_srt_single_node.py index 979f286daf..785442c671 100644 --- a/utils/test_srt_single_node.py +++ b/utils/test_srt_single_node.py @@ -453,3 +453,57 @@ def test_b300_keeps_agentic_and_explicit_collector_dispatch(tmp_path, collector) assert calls[-1][-2:] == ["bash", expected] assert "--jobid=42" in calls[-1] assert (tmp_path / "cancelled").read_text() == "42\n" + + +@pytest.mark.parametrize("eval_only,conc,capture", [ + ("false", "256", False), ("true", "128", False), ("true", "256", True), +]) +def test_qwen_b300_keeps_chat_template_and_eval_capture(point, eval_only, conc, capture): + path, recipe, env = point + model = "Qwen/Qwen3.5-397B-A17B-FP8" + recipe["model"]["path"] = f"hf:{model}" + recipe["resources"]["gpu_type"] = "b300" + recipe["benchmark"]["env"].update(MODEL=model, USE_CHAT_TEMPLATE="true") + path.write_text(yaml.safe_dump({"base": recipe})) + env = {**env, "MODEL": model, "CONC": conc, "EVAL_ONLY": eval_only, + "MAX_MODEL_LEN": "9472"} + argv = runtime_arguments(f"{path}:base", env) + actual = copy.deepcopy(recipe) + apply_overrides_to_recipe(actual, parse_overrides(argv[1::2], [])) + assert actual["benchmark"]["env"]["USE_CHAT_TEMPLATE"] == "true" + args = actual["roles"]["agg"]["args"] + assert args.get("log-requests", False) is capture + if capture: + assert args["log-requests-level"] == 3 + assert args["log-requests-format"] == "json" + assert actual["roles"]["agg"]["env"]["SGLANG_LOG_REQUEST_EXCEEDED_MS"] == "0" + assert args["context-length"] == 9472 + else: + assert "SGLANG_LOG_REQUEST_EXCEEDED_MS" not in actual["roles"]["agg"].get("env", {}) + recipe["benchmark"]["env"]["USE_CHAT_TEMPLATE"] = "false" + path.write_text(yaml.safe_dump({"base": recipe})) + with pytest.raises(ValueError, match="USE_CHAT_TEMPLATE"): + runtime_arguments(f"{path}:base", env) + + +@pytest.mark.parametrize("change", ["hardware", "model", "precision"]) +def test_qwen_chat_template_exception_does_not_change_neighbor_profiles(point, change): + path, recipe, env = point + model = "Qwen/Qwen3.5-397B-A17B-FP8" + recipe["model"]["path"] = f"hf:{model}" + recipe["resources"]["gpu_type"] = "b300" + recipe["benchmark"]["env"]["MODEL"] = model + env = {**env, "MODEL": model, "CONC": "256", "EVAL_ONLY": "true", "MAX_MODEL_LEN": "9472"} + if change == "hardware": + recipe["resources"]["gpu_type"] = "b200" + elif change == "model": + recipe["model"]["path"] = "hf:test/other" + recipe["benchmark"]["env"]["MODEL"] = env["MODEL"] = "test/other" + else: + recipe["model"]["precision"] = env["PRECISION"] = "fp4" + path.write_text(yaml.safe_dump({"base": recipe})) + argv = runtime_arguments(f"{path}:base", env) + actual = copy.deepcopy(recipe) + apply_overrides_to_recipe(actual, parse_overrides(argv[1::2], [])) + assert actual["benchmark"]["env"]["USE_CHAT_TEMPLATE"] == "false" + assert "log-requests" not in actual["roles"]["agg"]["args"]