From 96dccc915dd56fb6eb563bde6b323bc287ceff68 Mon Sep 17 00:00:00 2001 From: 1am9trash <1am9trash@gmail.com> Date: Sat, 26 Sep 2026 14:32:05 +0000 Subject: [PATCH 1/2] Rebase to main format of script --- .../dsv4/sglang/mi355x-fp4-mtp/agentic.yaml | 156 +++++++++++++++++- configs/amd-master.yaml | 5 +- perf-changelog.yaml | 11 ++ 3 files changed, 163 insertions(+), 9 deletions(-) diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml index b3aacda99..58449315b 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml @@ -1,12 +1,14 @@ # DeepSeek-V4-Pro-0813 AgentX on MI355X with SGLang DSpark. Low concurrency # keeps KV on GPU; higher concurrency extends it to host DRAM through HiCache, # and the widest band runs DP attention behind a cache-aware SGLang router. +# FP8 unified KV is on for every arm. BCG is limited to conc 1/4. Grouped ASM +# is on for TP-only arms at conc >= 16. EP8 DP arms use MegaMoE+EPLB. base: schema: 2 name: dsv4-fp4-mi355x-sglang-agentic model: path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 - container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260914 + container: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260926 precision: fp4 resources: gpu_type: mi355x @@ -62,6 +64,7 @@ base: SGLANG_DSV4_REASONING_EFFORT: high SGLANG_USE_ROCM700A: '0' SGLANG_HACK_FLASHMLA_BACKEND: unified_kv_triton + SGLANG_DSV4_UNIFIED_KV_FP8: '1' AITER_BF16_FP8_MOE_BOUND: '0' TORCH_BLAS_PREFER_HIPBLASLT: '1' HSA_NO_SCRATCH_RECLAIM: '0' @@ -91,6 +94,7 @@ override_tp8_c1: args: max-running-requests: 2 cuda-graph-max-bs-decode: 2 + cuda-graph-backend-prefill: breakable benchmark: env: CONC: '1' @@ -102,6 +106,7 @@ override_tp8_c4: args: max-running-requests: 8 cuda-graph-max-bs-decode: 8 + cuda-graph-backend-prefill: breakable benchmark: env: CONC: '4' @@ -113,6 +118,8 @@ override_tp8_c16: args: max-running-requests: 32 cuda-graph-max-bs-decode: 32 + env: + SGLANG_DSV4_GROUPED_ASM: '1' benchmark: env: CONC: '16' @@ -129,6 +136,8 @@ override_tp8_c32_hicache: hicache-write-policy: write_through hicache-io-backend: direct hicache-mem-layout: page_first_direct + env: + SGLANG_DSV4_GROUPED_ASM: '1' benchmark: env: CONC: '32' @@ -146,12 +155,15 @@ override_tp8_c48_hicache: hicache-write-policy: write_through hicache-io-backend: direct hicache-mem-layout: page_first_direct + env: + SGLANG_DSV4_GROUPED_ASM: '1' benchmark: env: CONC: '48' KV_OFFLOADING: dram AGENTIC_WARMUP_GRACE_PERIOD: '3600' +# EP1 DP-attention reference point (no MegaMoE). override_tp8_c128_dp_hicache: frontend: type: sglang-router @@ -181,6 +193,7 @@ override_tp8_c128_dp_hicache: tokenizer-worker-num: 8 stream-interval: 20 prefill-delayer-token-usage-low-watermark: 0.7 + load-balance-method: total_requests chunked-prefill-size: 65536 mem-fraction-static: 0.92 env: @@ -196,7 +209,65 @@ override_tp8_c128_dp_hicache: AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' AGENTIC_WARMUP_GRACE_PERIOD: '3600' -override_tp8_c256_dp_hicache: +# EP8 + MegaMoE + EPLB. Mori heap is outside mem-fraction. +override_tp8_c128_dp_ep8_megamoe_hicache: + frontend: + type: sglang-router + args: + policy: cache_aware + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + max-running-requests: 256 + cuda-graph-max-bs-decode: 128 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-prefill-delayer: true + enable-dp-attention-local-control-broadcast: true + tokenizer-worker-num: 8 + stream-interval: 20 + prefill-decode-interval: 20 + prefill-delayer-token-usage-low-watermark: 0.7 + load-balance-method: total_requests + chunked-prefill-size: 65536 + mem-fraction-static: 0.85 + moe-a2a-backend: megamoe + moe-dense-tp-size: 1 + disable-shared-experts-fusion: true + enable-eplb: true + eplb-rebalance-num-iterations: 200 + expert-distribution-recorder-mode: stat + env: + SGLANG_AMD_USE_FLYDSL_MEGA_MOE: '1' + SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR: '8192' + SGLANG_AMD_FLYDSL_MEGA_QUANT: a8w4 + MORI_SHMEM_HEAP_SIZE: '17179869184' + SGLANG_AITER_MEGA_RANK_SYNC: '1' + SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY: '1' + SGLANG_AITER_MEGA_EPLB_FUSED_MAP_RECORD: '1' + SGLANG_DP_USE_REDUCE_SCATTER: '0' + GPU_MAX_HW_QUEUES: '5' + benchmark: + env: + CONC: '128' + KV_OFFLOADING: dram + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c256_dp_ep8_megamoe_hicache: frontend: type: sglang-router args: @@ -219,20 +290,33 @@ override_tp8_c256_dp_hicache: hicache-io-backend: direct hicache-mem-layout: page_first_direct data-parallel-size: 8 + expert-parallel-size: 8 enable-dp-attention: true enable-dp-lm-head: true enable-prefill-delayer: true enable-dp-attention-local-control-broadcast: true tokenizer-worker-num: 8 stream-interval: 20 + prefill-decode-interval: 20 prefill-delayer-token-usage-low-watermark: 0.7 + load-balance-method: total_requests chunked-prefill-size: 65536 - mem-fraction-static: 0.92 + mem-fraction-static: 0.85 + moe-a2a-backend: megamoe + moe-dense-tp-size: 1 + disable-shared-experts-fusion: true + enable-eplb: true + eplb-rebalance-num-iterations: 200 + expert-distribution-recorder-mode: stat env: - SGLANG_SHARED_EXPERT_TP1: '1' - SGLANG_DP_SHARED_EXPERT_LOCAL: '1' - SGLANG_DP_USE_GATHERV: '1' - SGLANG_DP_USE_REDUCE_SCATTER: '1' + SGLANG_AMD_USE_FLYDSL_MEGA_MOE: '1' + SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR: '8192' + SGLANG_AMD_FLYDSL_MEGA_QUANT: a8w4 + MORI_SHMEM_HEAP_SIZE: '17179869184' + SGLANG_AITER_MEGA_RANK_SYNC: '1' + SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY: '1' + SGLANG_AITER_MEGA_EPLB_FUSED_MAP_RECORD: '1' + SGLANG_DP_USE_REDUCE_SCATTER: '0' GPU_MAX_HW_QUEUES: '5' benchmark: env: @@ -240,3 +324,61 @@ override_tp8_c256_dp_hicache: KV_OFFLOADING: dram AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c384_dp_ep8_megamoe_hicache: + frontend: + type: sglang-router + args: + policy: cache_aware + balance-abs-threshold: 32 + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + max-running-requests: 768 + cuda-graph-max-bs-decode: 128 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-prefill-delayer: true + enable-dp-attention-local-control-broadcast: true + tokenizer-worker-num: 8 + stream-interval: 20 + prefill-decode-interval: 15 + prefill-delayer-token-usage-low-watermark: 0.7 + load-balance-method: total_requests + chunked-prefill-size: 65536 + mem-fraction-static: 0.85 + moe-a2a-backend: megamoe + moe-dense-tp-size: 1 + disable-shared-experts-fusion: true + enable-eplb: true + eplb-rebalance-num-iterations: 200 + expert-distribution-recorder-mode: stat + env: + SGLANG_AMD_USE_FLYDSL_MEGA_MOE: '1' + SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR: '8192' + SGLANG_AMD_FLYDSL_MEGA_QUANT: a8w4 + MORI_SHMEM_HEAP_SIZE: '17179869184' + SGLANG_AITER_MEGA_RANK_SYNC: '1' + SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY: '1' + SGLANG_AITER_MEGA_EPLB_FUSED_MAP_RECORD: '1' + SGLANG_DP_USE_REDUCE_SCATTER: '0' + GPU_MAX_HW_QUEUES: '5' + benchmark: + env: + CONC: '384' + KV_OFFLOADING: dram + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index e08e74208..2d1ced6d4 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1361,7 +1361,7 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml } dsv4-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260914 + image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260926 model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1374,7 +1374,8 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: search-space: - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 16], spec-decoding: draft_model, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 48], spec-decoding: draft_model, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [128, 256], spec-decoding: draft_model, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [128], spec-decoding: draft_model, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [128, 256, 384], spec-decoding: draft_model, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml } # Upstream ROCm nightly rather than the deepseekv41-flash-0909 release tag: that # tag predates vllm-project/vllm#56503, which moves the mHC delayed pre block off diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0f361fd6d..d0996f1c1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8962,3 +8962,14 @@ - "旧镜像切出于 2026-09-16，新镜像领先 441 个提交，包含当前曲线所不含的三项 Kimi-K3 正确性修复：vllm-project/vllm#51483 不再将无状态的首个 chunk 当作 decode，vllm-project/vllm#57098 修复 Kimi-K3 reasoning parser，vllm-project/vllm#57430 支持 routed expert 量化。新镜像还包含本臂在并发大于 4 时均会使用的 ROCm CPU KV 卸载改动，包括 vllm-project/vllm#57160（ROCm CPU KV 卸载改用私有 pinned 张量）与 vllm-project/vllm#50045（卸载背压检测）。" - "This image bump does not change the DSpark draft model data type. Only the image: line changes and kimik3_fp4_mi355x_mtp.sh is unchanged; the draft loads unmodified from the published Inferact/Kimi-K3-DSpark checkpoint via --speculative-config (model=Inferact/Kimi-K3-DSpark, method=dspark). The only dtype in that speculative-config is kv_cache_dtype=fp8, which sets the draft KV-cache storage precision, not the draft weights. No flag overrides or re-quantizes the draft-model weights, so the draft dtype is preserved from its checkpoint across this re-sweep." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3419 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update image to lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260926." + - "Switch attention to fp8 unified KV; enable grouped decode on TP8-only arms at conc >= 16." + - "Enable breakable CUDA-graph prefill (BCG) at conc <= 4." + - "Enable EP8 + MegaMoE for DP-attention arms at conc >= 128 (including conc 384)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3430 From 63ccff206a78601753c239b78bad9e56df176bb6 Mon Sep 17 00:00:00 2001 From: 1am9trash <1am9trash@gmail.com> Date: Sat, 26 Sep 2026 14:34:06 +0000 Subject: [PATCH 2/2] Set shared-experts-fusion to false on megamoe --- .../srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml index 58449315b..183f0ec8c 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml @@ -244,6 +244,7 @@ override_tp8_c128_dp_ep8_megamoe_hicache: load-balance-method: total_requests chunked-prefill-size: 65536 mem-fraction-static: 0.85 + enforce-shared-experts-fusion: false moe-a2a-backend: megamoe moe-dense-tp-size: 1 disable-shared-experts-fusion: true @@ -302,6 +303,7 @@ override_tp8_c256_dp_ep8_megamoe_hicache: load-balance-method: total_requests chunked-prefill-size: 65536 mem-fraction-static: 0.85 + enforce-shared-experts-fusion: false moe-a2a-backend: megamoe moe-dense-tp-size: 1 disable-shared-experts-fusion: true @@ -360,6 +362,7 @@ override_tp8_c384_dp_ep8_megamoe_hicache: load-balance-method: total_requests chunked-prefill-size: 65536 mem-fraction-static: 0.85 + enforce-shared-experts-fusion: false moe-a2a-backend: megamoe moe-dense-tp-size: 1 disable-shared-experts-fusion: true