diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index 8b7d28794..ccba2f421 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -52,10 +52,12 @@ install_agentic_deps export VLLM_ROCM_AITER_MLA_ASM_PADDING=asm export VLLM_ROCM_USE_AITER=1 export SAFETENSORS_FAST_GPU=1 -export VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4=1 -export AITER_SITUV2_A8W4=1 +# Route the K3 SiTU MXFP4 MoE through the FlyDSL SiTUv2 a4w4 path (fp4 +# activations). vLLM exports AITER_SITUV2_A4W4=1 from this flag at init and +# clears any legacy AITER_SITUV2_A8W4 override; the tuned a4w4 dispatch needs +# AITER >= v0.1.20 (ROCm/aiter#4463), otherwise FlyDSL falls back to heuristics. +export VLLM_ROCM_USE_AITER_MOE_SITUV2=1 export AITER_BF16_FP8_MOE_BOUND=0 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 export AITER_QUICK_REDUCE_QUANTIZATION=INT4 # MEC FW <177 has an RCCL memory reclaim issue. @@ -110,7 +112,7 @@ case "${KV_OFFLOAD_BACKEND:-}" in lmcache) require_agentic_kv_offload_backend "$KV_OFFLOAD_BACKEND" - LMCACHE_VERSION=0.5.5.dev114+rocm7.2 + LMCACHE_VERSION=0.5.6.dev101+rocm7.2 LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/nightly-rocm" agentic_pip_install --quiet --no-cache-dir --no-deps \ @@ -217,7 +219,13 @@ case "$CONC" in GPU_MEM_UTIL=0.9 MAX_NUM_BATCHED_TOKENS=8192 ;; - 8|10) + 8) + SYNTHETIC_ACCEPT_LEN=3.84 + SPEC_NUM_TOKENS=7 + GPU_MEM_UTIL=0.9 + MAX_NUM_BATCHED_TOKENS=8192 + ;; + 10) SYNTHETIC_ACCEPT_LEN=3.36 SPEC_NUM_TOKENS=4 GPU_MEM_UTIL=0.9 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 543c40458..812aefec2 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -653,7 +653,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: dp-attn: false kimik3-fp4-mi355x-vllm-agentic-mtp: - image: vllm/vllm-openai-rocm:nightly-rocm100-af1c01499b289be555c475669ba50a88e96d846e + image: vllm/vllm-openai-rocm:nightly-rocm100-29468dde8b515031dc6d4d9d06bf0a2fa0442098 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:mi355x-amds @@ -662,11 +662,12 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: multinode: false scenarios: agentic-coding: - - dram-utilization: 0.60 + - dram-utilization: 0.50 search-space: - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1, 4] , spec-decoding: mtp} - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 10, 12, 14], spec-decoding: mtp } - - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [44, 48, 70], spec-decoding: mtp } + #- { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1, 4] , spec-decoding: mtp} + #- { tp: 8, ep: 1, dcp-size: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev101+rocm7.2" }, conc-list: [4, 8, 10, 12, 14], spec-decoding: mtp } + #- { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev101+rocm7.2" }, conc-list: [44, 48, 70], spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev101+rocm7.2" }, conc-list: [70, 80], spec-decoding: mtp } # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark # speculative decoding. Acceptance is pinned to the committed golden curve in diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1bc132cd3..4e7ccf26f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8950,3 +8950,13 @@ description: - "Update SGLang image from v0.5.19-cu130 to v0.5.20-cu130 and pass --cuda-graph-max-bs-decode after the alias removal." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3334 + +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Select the SiTUv2 a4w4 FlyDSL MoE path via the canonical VLLM_ROCM_USE_AITER_MOE_SITUV2=1 flag and drop the deprecated VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 alias and the legacy AITER_SITUV2_A8W4 override. vLLM exports AITER_SITUV2_A4W4=1 from this flag at init and clears any AITER_SITUV2_A8W4 (AITER checks A8W4 first); the tuned a4w4 dispatch needs AITER >= v0.1.20 (ROCm/aiter#4463), otherwise FlyDSL falls back to heuristic configs. This selects the fp4-activation MoE kernels and does not change the target or DSpark draft model weight precision." + - "Refresh the vLLM ROCm image to vllm/vllm-openai-rocm:nightly-rocm100-29468dde8b515031dc6d4d9d06bf0a2fa0442098 and bump the LMCache host KV tier to 0.5.6.dev101+rocm7.2." + - "Trim the search space to a single lmcache DRAM-offload arm at TP8/DCP8, conc [70, 80], dram-utilization 0.50, spec-decoding mtp; the earlier GPU-resident and vllm-simple arms are commented out for this re-sweep." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3447