diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh index 88311f2b04..942020cfef 100644 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh @@ -58,13 +58,15 @@ trap 'exit 143' TERM CACHE_ARGS=() if require_agentic_kv_offload_backend hicache; then HICACHE_RATIO="1.5" + HICACHE_SIZE="253" HICACHE_WRITE_POLICY="write_through" HICACHE_IO_BACKEND="kernel" HICACHE_MEM_LAYOUT="page_first" - echo "HiCache CPU tier: ratio=$HICACHE_RATIO, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT, dram_budget=${TOTAL_CPU_DRAM_GB} GB, tp=$TP" + echo "HiCache CPU tier: ratio=$HICACHE_RATIO, size=$HICACHE_SIZE GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT, dram_budget=${TOTAL_CPU_DRAM_GB} GB, tp=$TP" CACHE_ARGS=( --enable-hierarchical-cache --hicache-ratio "$HICACHE_RATIO" + --hicache-size "$HICACHE_SIZE" --hicache-write-policy "$HICACHE_WRITE_POLICY" --hicache-io-backend "$HICACHE_IO_BACKEND" --hicache-mem-layout "$HICACHE_MEM_LAYOUT" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 421d55c483..7b6562da7b 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -314,7 +314,7 @@ qwen3.5-fp4-mi355x-sglang-mtp: - { tp: 4, conc-start: 4, conc-end: 16, spec-decoding: mtp } qwen3.5-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915 + image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 model: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 model-prefix: qwen3.5 runner: cluster:mi355x-amds @@ -326,8 +326,8 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: - dram-utilization: 0.80 search-space: - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 36, 40] } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12] } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 20, 24, 28, 32, 36, 40] } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 52c1e01dc6..4215901a7c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8839,3 +8839,13 @@ - "Retain 8 SWA prefix tails per concurrency at C1/C2 after rejecting the 128-tail floor in matched canonical testing; use 32 per concurrency at C4 and above, reaching 640 at C20 within the static cache budget." - "Qualify supported DP8 attention and DP LM-head at C4/C8/C16/C20 with consistent-hash session routing and 64 SWA tails per rank; retain the complete plain TP8 curve and unchanged shipped precision, static budget and native context." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3345 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915 to lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924." + - "Move TP2 concurrency 16 from GPU-resident KV to HiCache and remove the TP2 GPU-resident concurrency-20 point. Keep TP4 GPU-resident [1, 4, 8, 12, 16]. The resulting TP2 grids are resident [1, 4, 8, 12] and HiCache [16, 20, 24, 28, 32, 36, 40]." + - "Pin the AgentX HiCache host pool with --hicache-size 253 (overrides --hicache-ratio)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3364