From 4fa2a2ffe45a668403bc3cec91b05a9d65811bc4 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 01:08:12 -0500 Subject: [PATCH 001/105] feat(srt): run single-node AgentX on native srt-slurm Single-node AgentX points with an srt-recipe now take the native single-node path fixed-sequence points already use. The adapter accepts AgentX points and DSpark speculation. Single- and multi-node AgentX share one client, benchmarks/srt_agentic.sh (moved from benchmarks/multi_node/agentic_srt.sh), which replays one CONC point or a CONC_LIST batch against the srt-slurm frontend and reads engine metrics from the workers behind a router frontend. The H200 DeepSeek-V4.1-Flash SGLang config is the first one ported. --- .../agentic/dsv4_fp4_mi355x_sglang-disagg.sh | 201 --------------- .../multi_node/amd_utils/trace_replay.sh | 4 +- .../configs/dsv41flash-h200-block32.sh | 5 + .../agentx/agg-b200-tp8-c1-mtp-hicache.yaml | 2 +- .../agentx/agg-b200-tp8-c4-mtp-hicache.yaml | 2 +- .../agentx/agg-b200-tp8-c8-mtp-hicache.yaml | 2 +- ...200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml | 2 +- ...b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml | 2 +- ...200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml | 2 +- .../sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml | 2 +- .../sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml | 2 +- ...gg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml | 4 +- ...gg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml | 4 +- ...g-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml | 4 +- ...g-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml | 4 +- .../agentx/agg-tp8-mtp-kvoffload.yaml | 2 +- .../mi355x-fp4/agentx/disagg-umbp-dspark.yaml | 232 ++++++++++++++++++ .../disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml | 2 +- .../disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml | 2 +- .../disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml | 2 +- .../disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml | 2 +- .../disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml | 2 +- .../disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml | 2 +- .../vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml | 2 +- .../gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml | 2 +- .../gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml | 2 +- .../vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml | 2 +- .../disagg-1p1d-dep8-dep8-c128-mtp3.yaml | 2 +- .../disagg-1p1d-dep8-dep8-c256-mtp3.yaml | 2 +- .../agentx/disagg-1p1d-dep8-dep8-mtp.yaml | 2 +- .../disagg-2p1d-dep8-dep12-c576-mtp3.yaml | 2 +- .../disagg-2p1d-dep8-dep16-c512-mtp3.yaml | 2 +- .../agentx/disagg-2p1d-dep8-dep8-mtp.yaml | 2 +- .../vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml | 2 +- .../vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml | 2 +- .../disagg-1p1d-dep4-dep16-c128-mtp.yaml | 2 +- .../disagg-1p1d-dep4-dep16-c256-mtp.yaml | 2 +- .../disagg-1p1d-dep4-dep8-c256-mtp.yaml | 2 +- .../disagg-1p1d-dep8-dep16-c512-mtp.yaml | 2 +- .../agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml | 2 +- .../agentx/disagg-1p6d-dep4-tp4.yaml | 6 +- .../disagg-2p1d-dep8-dep12-c1152-mtp.yaml | 2 +- .../disagg-2p1d-dep8-dep16-c1024-mtp.yaml | 2 +- .../disagg-4p1d-dep4-dep8-24-c4096.yaml | 6 +- .../b200-fp8/agentx/disagg-1p1d-tp8-mtp.yaml | 2 +- .../b200-fp4/agentx/agg-tp8-c1-mtp.yaml | 4 +- .../b200-fp4/agentx/agg-tp8-c4-mtp.yaml | 4 +- .../b200-fp4/agentx/agg-tp8-c8-mtp.yaml | 4 +- .../agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml | 2 +- .../agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml | 2 +- .../gb200-fp4/agentx/agg-tp8-c2-mtp.yaml | 2 +- .../gb200-fp4/agentx/agg-tp8-c4-mtp.yaml | 2 +- .../gb200-fp4/agentx/agg-tp8-c8-mtp.yaml | 2 +- .../glm5.2/sglang/gb200-fp4/agentx/agg.yaml | 2 +- .../agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml | 2 +- .../agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml | 2 +- .../disagg-2p1d-dep8-dep16-c128-mtp.yaml | 2 +- .../gb200-fp4/agentx/disagg-mtp-variants.yaml | 2 +- .../glm5.2/sglang/gb300-fp4/agentx/agg.yaml | 4 +- .../gb300-fp4/agentx/disagg-mtp-variants.yaml | 2 +- .../h200-fp8/agentx/agg-h200-tp8-mtp-c12.yaml | 2 +- .../h200-fp8/agentx/agg-h200-tp8-mtp-c16.yaml | 2 +- .../h200-fp8/agentx/agg-h200-tp8-mtp-c2.yaml | 2 +- .../h200-fp8/agentx/agg-h200-tp8-mtp-c4.yaml | 2 +- .../h200-fp8/agentx/agg-h200-tp8-mtp-c8.yaml | 2 +- ...disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml | 3 +- .../agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml | 2 +- .../agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml | 2 +- .../agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml | 2 +- .../agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml | 2 +- .../agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml | 2 +- .../disagg-4p1d-dep8-c227-b16-mtp3.yaml | 2 +- .../disagg-5p1d-dep16-c260-b16-mtp3.yaml | 2 +- .../agentx/agg-tp8pp2-mooncake-c1.yaml | 2 +- .../agentx/agg-tp8pp2-mooncake-c14.yaml | 2 +- .../agentx/agg-tp8pp2-mooncake-c24.yaml | 2 +- .../agentx/agg-tp8pp2-mooncake-c4.yaml | 2 +- .../agentx/agg-tp8pp2-mooncake-c48.yaml | 2 +- .../agentx/agg-tp8pp2-mooncake-c8.yaml | 2 +- .../agentx/agg-tp8pp2-mooncake-c96.yaml | 2 +- .../agg-dcp16-dspark4-maxseq2-mooncake.yaml | 2 +- .../agentx/agg-dcp16-nospec-mooncake.yaml | 2 +- .../agentx/agg-dep16-vllm-simple-offload.yaml | 2 +- .../vllm/gb200-fp4/agentx/agg-dep16.yaml | 2 +- .../gb200-fp4/agentx/agg-tep16-balanced.yaml | 2 +- .../gb200-fp4/agentx/agg-tp16-latency.yaml | 2 +- .../agentx/agg-tp8pp2-mooncake-c16.yaml | 2 +- .../agentx/agg-tp8pp2-mooncake-c32.yaml | 2 +- .../agentx/agg-tp8pp2-mooncake-c48.yaml | 2 +- .../agentx/agg-tp8pp2-mooncake-c72.yaml | 2 +- .../agentx/agg-tp8pp2-mooncake-c96.yaml | 2 +- .../agentx/agg-dcp8-dspark4-mooncake.yaml | 2 +- .../agg-dcp8-dspark7-maxseq2-mooncake.yaml | 2 +- ...isagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml | 5 +- ...isagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml | 5 +- ...isagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml | 5 +- ...isagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml | 5 +- .../agentx/agg-tp16dp2ep32-latency.yaml | 2 +- .../agentx/agg-tp8dp4ep32-balanced.yaml | 5 +- .../agentx/agg-tp8dp4ep32-vllm-simple.yaml | 5 +- .../agentx/agg-tp4-c10-b10-eagle3.yaml | 2 +- .../agentx/agg-tp4-c15-b15-eagle3.yaml | 2 +- .../agentx/agg-tp4-c20-b20-eagle3.yaml | 2 +- .../agentx/agg-tp4-c25-b25-eagle3.yaml | 2 +- .../agentx/agg-tp4-c30-b30-eagle3.yaml | 2 +- .../agentx/agg-tp4-c40-b40-eagle3.yaml | 2 +- .../agentx/agg-tp4-c5-b5-eagle3.yaml | 2 +- .../agentx/agg-dep4-vllm-simple.yaml | 2 +- .../vllm/gb200-fp4/agentx/agg-dep4.yaml | 2 +- .../vllm/gb200-fp4/agentx/agg-dep8.yaml | 2 +- .../agentx/agg-tp4-nightly-native.yaml | 2 +- .../agg-tp4-vllm-simple-nightly-native.yaml | 2 +- .../gb200-fp4/agentx/agg-tp4-vllm-simple.yaml | 2 +- .../vllm/gb200-fp4/agentx/agg-tp4.yaml | 2 +- .../agentx/agg-tp8-nightly-native.yaml | 2 +- .../agentx/disagg-1p1d-dep8-dep4.yaml | 2 +- .../agentx/disagg-1p1d-tp4-tp4-c24.yaml | 2 +- .../agentx/disagg-1p1d-tp4-tp8-c1.yaml | 2 +- .../agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml | 2 +- .../agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml | 2 +- .../agentx/disagg-1p1d-tep4-tp4-c1.yaml | 2 +- .../disagg-1p1d-tp2-tp4-c20-c24-eval.yaml | 2 +- .../agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml | 2 +- .../agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml | 2 +- .../agentx/disagg-1p3d-dep4-tp4-c24.yaml | 2 +- .../agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml | 2 +- .../agentx/disagg-1p3d-tp2-tp2-c48.yaml | 2 +- .../agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml | 2 +- .../agentx/disagg-2p5d-tp2-tp2-c120.yaml | 2 +- ...sagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml | 2 +- ...sagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml | 2 +- ...sagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml | 2 +- ...4-d-tp4-hicache-c48-write-through-mtp.yaml | 2 +- ...4-d-tp4-hicache-c64-write-through-mtp.yaml | 2 +- ...isagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml | 2 +- ...2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml | 2 +- ...2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml | 2 +- ...2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml | 2 +- ...2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml | 2 +- ...2-colocated-c56-replayssm-mtp-hicache.yaml | 2 +- ...p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml | 2 +- ...p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml | 2 +- ...1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml | 2 +- ...-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml | 2 +- .../agentx/agg-tp2ep2-mtp-hicache-cap48.yaml | 2 +- .../agg-tp2ep2-mtp-hicache-k3-baseline.yaml | 2 +- .../agentx/agg-tp2ep2-mtp-hicache.yaml | 2 +- .../gb200-fp4/agentx/agg-tp4-mtp-hicache.yaml | 2 +- .../gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml | 2 +- .../gb200-fp4/agentx/agg-tp4-mtp-parity.yaml | 2 +- .../sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml | 2 +- .../agg-tp2-c1-mtp-hicache-jid2530006.yaml | 2 +- .../agg-tp2-c24-mtp-hicache-jid2530012.yaml | 2 +- .../agg-tp2-c32-mtp-hicache-jid2530013.yaml | 2 +- .../agg-tp2-c40-mtp-hicache-jid2530015.yaml | 2 +- ...-c44-b1-mtp-hicache-nightly-c20260831.yaml | 2 +- ...-c44-b2-mtp-hicache-nightly-c20260831.yaml | 2 +- .../agg-tp2-c48-mtp-hicache-jid2530017.yaml | 2 +- .../agg-tp2-c52-mtp-hicache-jid2527406.yaml | 2 +- .../agg-tp2-c64-mtp-hicache-jid2527410.yaml | 2 +- ...8-c7-b1-mtp-hicache-nightly-c20260831.yaml | 2 +- ...p2-c72-mtp-hicache-session-jid2527415.yaml | 2 +- ...4-c128-mtp-hicache-session-jid2527417.yaml | 2 +- ...p4-c16-mtp-hicache-session-jid2530027.yaml | 2 +- ...p4-c32-mtp-hicache-session-jid2530028.yaml | 2 +- ...p4-c64-mtp-hicache-session-jid2530029.yaml | 2 +- ...tp4-c8-mtp-hicache-session-jid2530030.yaml | 2 +- ...p4-c96-mtp-hicache-session-jid2527409.yaml | 2 +- ...p4-c704-mtp-hicache-nightly-c20260831.yaml | 2 +- ...p4-c565-mtp-hicache-nightly-c20260831.yaml | 2 +- ...g-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml | 2 +- ...gg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml | 2 +- ...g-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml | 2 +- ...2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml | 2 +- ...3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml | 2 +- ...3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml | 2 +- .../sglang/b200-fp4-mtp/agentic.yaml | 76 ++++++ .../sglang/h200-fp4-mtp/agentic.yaml | 101 ++++++++ .../qwen3.5/sglang/h200-fp8-mtp/agentic.yaml | 77 ++++++ .../qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml | 75 ------ .../sglang/h200-fp8-mtp/agentic.yaml | 61 +++++ .../agentic_srt.sh => srt_agentic.sh} | 35 ++- configs/amd-master.yaml | 64 +++-- configs/nvidia-master.yaml | 14 +- docs/eval-agentx-procedures.md | 2 +- docs/eval-agentx-procedures_zh.md | 2 +- infx/matrix/generate.py | 2 + infx/matrix/validation.py | 2 + infx/srt_slurm/single_node.py | 41 ++-- runners/launch_b200-cw.sh | 2 +- runners/launch_b200-nb.sh | 2 +- runners/launch_b200-nscale-slurm.sh | 4 +- runners/launch_b300-dsxe.sh | 2 +- runners/launch_h100-cw.sh | 2 +- runners/launch_h100-dgxc-slurm.sh | 2 +- runners/launch_h200-cw.sh | 2 +- runners/launch_h200-dgxc-slurm.sh | 3 +- runners/launch_mi300x-amd.sh | 2 +- runners/launch_mi325x-amds.sh | 2 +- runners/launch_mi355x-amds.sh | 2 +- runners/slurm_utils.sh | 6 +- utils/evals/test_run_eval_dispatch.py | 2 +- utils/test_srt_single_node.py | 2 +- 203 files changed, 871 insertions(+), 545 deletions(-) delete mode 100755 benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h200-block32.sh create mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml create mode 100644 benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml delete mode 100644 benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml create mode 100644 benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml rename benchmarks/{multi_node/agentic_srt.sh => srt_agentic.sh} (76%) diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index a69ca7cc4d..0000000000 --- a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env bash - -# Agentic trace-replay recipe for a disaggregated SGLang server on MI355X -# (DeepSeek-V4-Pro FP4, 1P1D TP8). Driven by environment variables; submits a SLURM -# job via submit.sh. - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../../benchmark_lib.sh" --validation-only - -check_env_vars \ - TIME_LIMIT MODEL_PREFIX PRECISION RESULT_FILENAME DURATION \ - MAX_MODEL_LEN DISABLE_CUSTOM_ALL_REDUCE KV_OFFLOADING MORI_IO_SQ_BACKOFF_TIMEOUT_US \ - MORI_IO_QP_MAX_SEND_WR PREFILL_ROUTER_POLICY ENABLE_METRICS DECODE_MTP_SIZE - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - DURATION \ - KV_OFFLOADING \ - IS_AGENTIC \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -export MODEL_PREFIX -export PRECISION -export RESULT_FILENAME - -export DURATION -export MAX_MODEL_LEN - -# --disable-custom-all-reduce avoids a known aiter fault on MI355X. -export DISABLE_CUSTOM_ALL_REDUCE - -# ── KV cache offloading ── -# KV_OFFLOADING=none | dram (passed from YAML). -# KV_OFFLOAD_BACKEND selects the backend when offloading is on: -# hicache GPU + CPU-DRAM host pool (HICACHE_TIER L2), optionally + a -# Mooncake L3 store (HICACHE_TIER L3). The tunables below. -# umbp-linker UMBP as a DIRECT external store for the unified radix tree, -# with NO host cache tier in between. A different sglang code -# path, not a variation of HiCache -- sglang rejects the two -# together -- so it reads NONE of the HICACHE_*/MC_* tunables -# and takes UMBP_* instead (block further down). Implemented in -# amd_utils/server_sglang.sh; prefill-side only, like HiCache -# on this path, and dp-attn: true only. -export KV_OFFLOADING -if [[ "$KV_OFFLOADING" != "none" ]]; then - check_env_vars KV_OFFLOAD_BACKEND -fi -if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; then - check_env_vars \ - HICACHE_TIER HICACHE_HOST_POOL_COUNT HICACHE_PAGE_SIZE HICACHE_RATIO HICACHE_MEM_LAYOUT \ - HICACHE_IO_BACKEND HICACHE_WRITE_POLICY HICACHE_PREFETCH_POLICY MC_MASTER_PORT MC_METADATA_PORT \ - MC_METRICS_PORT MC_MASTER_THREADS MC_EVICTION_HIGH_WATERMARK MC_PROTOCOL \ - MC_GLOBAL_SEG - export HICACHE_TIER - export HICACHE_HOST_POOL_COUNT - # DSV4 uses page-size 256 (set in models.yaml); HiCache must match. - export HICACHE_PAGE_SIZE - export HICACHE_RATIO - # server_sglang.sh prefers --hicache-size over --hicache-ratio when TOTAL_CPU_DRAM_GB - # is set; opt out via FORCE_HICACHE_RATIO rather than unsetting TOTAL_CPU_DRAM_GB, - # which benchmark_lib.sh also requires client-side when KV_OFFLOADING=dram. - export FORCE_HICACHE_RATIO=1 - - if [[ "${HICACHE_TIER^^}" == "L3" ]]; then - export HICACHE_MEM_LAYOUT - export HICACHE_IO_BACKEND - export HICACHE_WRITE_POLICY - if [[ -z "${HICACHE_STORAGE_BACKEND:-}" ]]; then - export HICACHE_STORAGE_BACKEND=mooncake - fi - else - export HICACHE_MEM_LAYOUT - export HICACHE_IO_BACKEND - export HICACHE_WRITE_POLICY - export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" - fi - export HICACHE_PREFETCH_POLICY - # Shared nodes: use non-default Mooncake ports to avoid collisions. - export MC_MASTER_PORT - export MC_METADATA_PORT - export MC_METRICS_PORT - export MC_MASTER_THREADS - export MC_EVICTION_HIGH_WATERMARK - export MC_PROTOCOL - export MC_GLOBAL_SEG - export MC_DEVICE="${MC_DEVICE:-}" - export MC_MASTER_ADDR="${MC_MASTER_ADDR:-}" - export MC_METADATA_SERVER="${MC_METADATA_SERVER:-}" -fi - -# ── UMBP direct-linker tunables ── -# Only read when KV_OFFLOAD_BACKEND is a umbp-linker* arm. Defaults live in -# server_sglang.sh; these exports exist so the values are visible in the -# recipe (and in the commands dump) rather than buried, and so job.slurm has -# something to forward. -# UMBP_DRAM_BYTES NODE total for the tier, on the prefill node only. -# 1.5 TB matches the single-node linker arms, so a PD -# number can be read against them directly. Guarded in -# server_sglang.sh against UMBP_DRAM_CEILING_GB. -# UMBP_DRAM_CEILING_GB tier-size ceiling; defaults to half of MemTotal. -# UMBP_MAX_TOTAL_TOKENS optional device KV pool cap. UNSET on purpose: the -# linker is compared against the HiCache control at an -# IDENTICAL profiled pool, not at a capped one. -# UMBP_SA_WAIT_SECONDS ceiling for each of the three server-readiness waits -# (socket -> data plane -> host memory registered for -# GPU access). A 1.5 TB tier can take many minutes to -# register on a node holding a lot of page cache. -# UMBP_DRAM_USE_HUGEPAGES on by default; 2 MiB pages keep tier registration -# time predictable. The run fails rather than falling -# back to 4 KiB pages. Set 0 to opt out. -if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == umbp-linker* ]]; then - export UMBP_DRAM_BYTES="${UMBP_DRAM_BYTES:-1500000000000}" - export UMBP_DRAM_CEILING_GB="${UMBP_DRAM_CEILING_GB:-}" - export UMBP_DRAM_USE_HUGEPAGES="${UMBP_DRAM_USE_HUGEPAGES:-1}" - export UMBP_SA_WAIT_SECONDS="${UMBP_SA_WAIT_SECONDS:-1800}" - export UMBP_SA_WAIT_REGISTERED="${UMBP_SA_WAIT_REGISTERED:-1}" - export MORI_UMBP_LOG_LEVEL="${MORI_UMBP_LOG_LEVEL:-info}" - # Rank 0 opens the barrier only after the UMBP tier is registered, so the - # barrier must outlast UMBP_SA_WAIT_SECONDS. - export CONTAINER_BARRIER_TIMEOUT="${CONTAINER_BARRIER_TIMEOUT:-$((UMBP_SA_WAIT_SECONDS + 600))}" -fi - -# ── MoRIIO RDMA Send Queue tuning ── -export MORI_IO_SQ_BACKOFF_TIMEOUT_US -export MORI_IO_QP_MAX_SEND_WR - -export PREFILL_ROUTER_POLICY -export ENABLE_METRICS - -export DECODE_MTP_SIZE - -if [[ "${PREFILL_EP}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch the job. CONC_LIST is space-delimited in YAML; submit.sh wants 'x'. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/amd_utils/trace_replay.sh b/benchmarks/multi_node/amd_utils/trace_replay.sh index c1d5f76859..204e872cab 100644 --- a/benchmarks/multi_node/amd_utils/trace_replay.sh +++ b/benchmarks/multi_node/amd_utils/trace_replay.sh @@ -73,7 +73,7 @@ PORT="${ROUTER_PORT}" check_env_vars DURATION RESULT_FILENAME FLUSH_DRAIN_TIMEOUT CLEAR_CACHE_BETWEEN_CONC export MODEL DURATION MAX_MODEL_LEN # The workflow guard / upload steps expect one "${RESULT_FILENAME}_conc.json" per -# concurrency, so each conc below is suffixed with _conc (as agentic_srt.sh does). +# concurrency, so each conc below is suffixed with _conc (as srt_agentic.sh does). RESULT_FILENAME_BASE="${RESULT_FILENAME}" mkdir -p "$RESULT_DIR" @@ -103,7 +103,7 @@ for max_concurrency in "${chosen_concurrencies[@]}"; do # benchmark-multinode-tmpl.yml expects the per-conc nesting (LOGS/agentic/conc_*/...) # even though CI runs one concurrency per job; nesting also keeps local multi-conc - # sweeps from overwriting each other (same layout as agentic_srt.sh). + # sweeps from overwriting each other (same layout as srt_agentic.sh). CONC_RESULT_DIR="$RESULT_DIR/conc_${max_concurrency}" mkdir -p "$CONC_RESULT_DIR" diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h200-block32.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h200-block32.sh new file mode 100755 index 0000000000..3d311f530c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h200-block32.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Install the measured H200 DeepSeek-V4.1-Flash block-32 tilings into the worker's SGLang. +set -euo pipefail +agentic=/infmax-workspace/benchmarks/single_node/agentic +python3 "$agentic/install_h200_block32_configs.py" "$agentic/kernel_configs/h200_dsv41_block32" /logs "$DSV41_BLOCK32_TP" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c1-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c1-mtp-hicache.yaml index f2368fa8a8..c24d199b7c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c1-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c1-mtp-hicache.yaml @@ -122,7 +122,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c4-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c4-mtp-hicache.yaml index 5cc7a371c4..06b0f96500 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c4-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c4-mtp-hicache.yaml @@ -122,7 +122,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c8-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c8-mtp-hicache.yaml index e1f28fa87d..403e23ca15 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c8-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c8-mtp-hicache.yaml @@ -122,7 +122,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml index 305bf5d7ef..10a55422a4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml @@ -221,7 +221,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml index 16f1ed6c8e..bfe784c531 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml @@ -221,7 +221,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml index 4ac6c19f63..ea011fe829 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml @@ -226,7 +226,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml index c49541b92b..f270e4ffc8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml @@ -116,7 +116,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml index 0aea65fbef..c633f7506f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml @@ -116,7 +116,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml index 15885fb9af..03bd3727d5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml @@ -7,7 +7,7 @@ name: "disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload" # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized # from the Pareto run. -# Concurrency is exported into agentic_srt.sh from the master-config conc-list. +# Concurrency is exported into srt_agentic.sh from the master-config conc-list. model: path: "deepseek-v4-pro-0813" @@ -221,7 +221,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml index 00ce318b0d..2ae7922cec 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml @@ -6,7 +6,7 @@ name: "disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload" # # DEP8-prefill variant aligned with the measured Pareto point: prefill uses # tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. -# Concurrency is exported into agentic_srt.sh +# Concurrency is exported into srt_agentic.sh # from the master-config conc-list. model: @@ -223,7 +223,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml index f0e66a32d8..95bb232ce5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml @@ -7,7 +7,7 @@ name: "disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload" # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized # from the Pareto run. -# Concurrency is exported into agentic_srt.sh from the master-config conc-list. +# Concurrency is exported into srt_agentic.sh from the master-config conc-list. model: path: "deepseek-v4-pro-0813" @@ -221,7 +221,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml index e569bd41cd..f0de0d1381 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml @@ -7,7 +7,7 @@ name: "disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload" # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized # from the Pareto run. -# Concurrency is exported into agentic_srt.sh from the master-config conc-list. +# Concurrency is exported into srt_agentic.sh from the master-config conc-list. model: path: "deepseek-v4-pro-0813" @@ -225,7 +225,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml index d1b42f5e6b..d3c670c0f0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml @@ -127,7 +127,7 @@ benchmark: type: custom # Replaced with CONC_LIST by the launcher before submission. concurrencies: [1] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml new file mode 100644 index 0000000000..4429f029e3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml @@ -0,0 +1,232 @@ +# DeepSeek-V4-Pro-0813 AgentX on MI355X: 1P1D SGLang disaggregation over MoRI +# with DSpark. Every arm offloads prefill KV to UMBP as a direct external store +# for the unified radix tree (no host cache tier), backed by a 1.5 TB +# hugepage DRAM tier on the prefill node. +base: + name: mi355x-dsv4-pro-0813-agentx-umbp + model: + path: DeepSeek-V4-Pro-0813 + container: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260923 + precision: fp4 + slurm: + time_limit: "08:00:00" + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang-router + enable_multiple_frontends: false + args: + policy: consistent_hashing + dp-aware: true + cache-threshold: 0.3 + balance-abs-threshold: 2 + balance-rel-threshold: 1.1 + disable-circuit-breaker: true + health-failure-threshold: 100 + health-check-timeout-secs: 600 + health-check-interval-secs: 30 + engine: sglang + # A recipe host_setup replaces the cluster default, so run its fabric, GPU and + # hugepage-reclaim checks first. Then reserve the UMBP tier's hugepages on the + # prefill node (the allocation's first node in 1P1D) so the tier never falls + # back to 4 KiB pages; teardown returns them for the next job. + host_setup: + nodes: workers + timeout_seconds: 1800 + commands: + - IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" + - >- + if [[ "$(hostname -s)" == "$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)" ]]; then + page_kb=$(awk '/^Hugepagesize:/ {print $2}' /proc/meminfo); + pages=$(( (1500000000000 / (page_kb * 1024) + 1) * 102 / 100 )); + HUGEPAGES_TARGET=$pages HUGEPAGE_GROW_TOLERANCE_PCT=1 + bash "${GITHUB_WORKSPACE}/benchmarks/multi_node/amd_utils/helpers/hugepage_sanity.sh"; fi + teardown: + - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/benchmarks/multi_node/amd_utils/helpers/hugepage_sanity.sh" + # The per-allocation /logs mount is shared by the service and prefill + # containers, so the socket cannot collide with another job's UMBP. + services: + - name: umbp + type: generic + placement: + node: prefill + start: before_workers + critical: true + inherit_discovery_env: false + command: + - /sgl-workspace/mori/python/mori/umbp_standalone_server + - unix:///logs/umbp.sock + env: + HSA_NO_SCRATCH_RECLAIM: "0" + GPU_MAX_HW_QUEUES: "5" + UMBP_DRAM_CAPACITY: "1500000000000" + UMBP_DRAM_USE_HUGEPAGES: "1" + UMBP_SSD_ENABLED: "0" + MORI_UMBP_LOG_LEVEL: info + preamble: | + host_gb=$(awk '/^MemTotal:/ {printf "%d", $2 / 1000000}' /proc/meminfo) + if (( UMBP_DRAM_CAPACITY / 1000000000 > host_gb / 2 )); then + echo "UMBP capacity exceeds half of host memory" >&2 + exit 1 + fi + export LD_LIBRARY_PATH="/sgl-workspace/mori/python/mori:${LD_LIBRARY_PATH}" + readiness: + # Wait for GPU registration of the tier, not merely a listening socket. + log: + pattern: host memory registered for GPU access + timeout_seconds: 5400 + interval_seconds: 1 + roles: + prefill: + nodes: 1 + workers: 1 + env: &environment + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub + PYTHONDONTWRITEBYTECODE: "1" + PYTHONUNBUFFERED: "1" + PYTHONPATH: /sgl-workspace/aiter + TORCH_NCCL_BLOCKING_WAIT: "1" + NCCL_BLOCKING_WAIT: "1" + NCCL_IB_HCA: rdma3,rdma0,rdma2,rdma1,rdma7,rdma4,rdma6,rdma5 + GLOO_SOCKET_IFNAME: eno0 + NCCL_SOCKET_IFNAME: eno0 + AITER_LOG_LEVEL: ERROR + HSA_NO_SCRATCH_RECLAIM: "0" + GPU_MAX_HW_QUEUES: "2" + SGLANG_AITER_MLA_PERSIST: "0" + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_MORI_DISPATCH_DTYPE: auto + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "4096" + SGLANG_MORI_RECV_BOUND: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_USE_AITER: "1" + SGLANG_USE_ROCM700A: "0" + SGLANG_HACK_FLASHMLA_BACKEND: unified_kv_triton + SGLANG_OPT_USE_AITER_INDEXER: "true" + SGLANG_OPT_FP8_WO_A_FUSED_INVROPE: "1" + SGLANG_OPT_USE_AITER_BATCHED_GEMM: "1" + AITER_BF16_FP8_MOE_BOUND: "0" + TORCH_BLAS_PREFER_HIPBLASLT: "1" + IBDEVICES: rdma3,rdma0,rdma2,rdma1,rdma7,rdma4,rdma6,rdma5 + MORI_RDMA_TC: "104" + MORI_IO_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "500000" + MORI_IO_QP_MAX_SEND_WR: "32767" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_IO_TC_DISABLE: "0" + MORI_SHMEM_MODE: ISOLATION + MORI_EP_LAUNCH_CONFIG_MODE: AUTO + MORI_APP_LOG_LEVEL: WARNING + SGLANG_MORI_COMBINE_DTYPE: auto + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + MORI_MAX_DISPATCH_TOKENS_PREFILL: "8192" + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_ENABLE_OVERLAP_PLAN_STREAM: "0" + SGLANG_LOG_MS: "true" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_ROUTER_STDOUT_LOGS: "0" + UMBP_STANDALONE_ADDRESS: unix:///logs/umbp.sock + args: &server + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tp-size: 8 + enable-deepseek-v4-fp4-indexer: true + watchdog-timeout: 3600 + load-balance-method: round_robin + kv-cache-dtype: fp8_e4m3 + attention-backend: dsv4 + page-size: 256 + swa-full-tokens-ratio: 0.1 + enforce-shared-experts-fusion: true + tool-call-parser: deepseekv4 + reasoning-parser: deepseek-v4 + disaggregation-transfer-backend: mori + disaggregation-ib-device: rdma3,rdma0,rdma2,rdma1,rdma7,rdma4,rdma6,rdma5 + tokenizer-worker-num: 8 + stream-interval: 20 + log-level: info + log-level-http: error + speculative-algorithm: DSPARK + speculative-eagle-topk: 1 + speculative-num-steps: 1 + context-length: 1048576 + enable-metrics: true + disable-cuda-graph: true + disable-cuda-graph: true + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mori + hicache-storage-backend-extra-config: '{"standalone_startup_timeout_ms":120000}' + enable-cache-report: true + # Optimistic prefill (sgl-project/sglang#38978) is wired for the linker only. + optimistic-prefill-attempts: 2 + decode: + nodes: 1 + workers: 1 + env: + <<: *environment + # Decode runs no UMBP tier; offload is prefill-side. + UMBP_STANDALONE_ADDRESS: "" + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "128" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + <<: *server + disable-cuda-graph: false + sbatch_directives: + cpus-per-task: "128" + mem: "0" + srun_options: + mem: "0" + container-writable: "" + container-remap-root: "" + health_check: + max_attempts: 720 + interval_seconds: 5 + benchmark: + type: custom + container_image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907 + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + # The MI355X launcher collects results from the job workspace. + RESULT_DIR: /infmax-workspace/LOGS/agentic + AGENTIC_OUTPUT_DIR: /infmax-workspace + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache/hub + TOKENIZERS_PARALLELISM: "false" + TRANSFORMERS_VERBOSITY: error + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + +# One variant per point; each point runs in its own allocation. Admission is +# 2x CONC on both roles. Decode captures graphs up to 2x CONC, or CONC/4 per +# rank under DP attention. DSpark draft length is 6 at c4/c16 (the golden AL +# peak) and 3 above; MoRI decode dispatch scales with it (64 x (draft + 1)). +override_c4: + roles: {prefill: {gpus: 4, args: {tp-size: 4, mem-fraction-static: 0.86, chunked-prefill-size: 16384, max-running-requests: 8, speculative-dspark-block-size: 6, speculative-num-draft-tokens: 7}}, decode: {gpus: 4, env: {MORI_MAX_DISPATCH_TOKENS_DECODE: "448"}, args: {tp-size: 4, mem-fraction-static: 0.86, max-running-requests: 8, cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8], speculative-dspark-block-size: 6, speculative-num-draft-tokens: 7}}} + benchmark: {env: {CONC: '4'}} +override_c16: + roles: {prefill: {gpus: 4, args: {tp-size: 4, mem-fraction-static: 0.86, chunked-prefill-size: 16384, max-running-requests: 32, speculative-dspark-block-size: 6, speculative-num-draft-tokens: 7}}, decode: {gpus: 8, env: {MORI_MAX_DISPATCH_TOKENS_DECODE: "448"}, args: {tp-size: 8, mem-fraction-static: 0.86, max-running-requests: 32, cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], speculative-dspark-block-size: 6, speculative-num-draft-tokens: 7}}} + benchmark: {env: {CONC: '16'}} +override_c32: + roles: {prefill: {gpus: 4, args: {tp-size: 4, mem-fraction-static: 0.86, chunked-prefill-size: 16384, max-running-requests: 64, speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}, decode: {gpus: 8, env: {MORI_MAX_DISPATCH_TOKENS_DECODE: "256"}, args: {tp-size: 8, mem-fraction-static: 0.86, max-running-requests: 64, cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64], speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}} + benchmark: {env: {CONC: '32'}} +override_c48: + roles: {prefill: {gpus: 4, args: {tp-size: 4, mem-fraction-static: 0.86, chunked-prefill-size: 16384, max-running-requests: 96, speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}, decode: {gpus: 8, env: {MORI_MAX_DISPATCH_TOKENS_DECODE: "256"}, args: {tp-size: 8, mem-fraction-static: 0.86, max-running-requests: 96, cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96], speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}} + benchmark: {env: {CONC: '48'}} +override_c192: + roles: {prefill: {gpus: 8, env: {GPU_MAX_HW_QUEUES: "5", SGLANG_SHARED_EXPERT_TP1: "1", SGLANG_DP_SHARED_EXPERT_LOCAL: "1", SGLANG_DP_USE_GATHERV: "1", SGLANG_DP_USE_REDUCE_SCATTER: "1", }, args: {tp-size: 8, dp-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, swa-full-tokens-ratio: 0.15, mem-fraction-static: 0.92, chunked-prefill-size: 65536, enable-two-batch-overlap: false, max-running-requests: 384, speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}, decode: {gpus: 8, env: {GPU_MAX_HW_QUEUES: "5", SGLANG_SHARED_EXPERT_TP1: "1", SGLANG_DP_SHARED_EXPERT_LOCAL: "1", SGLANG_DP_USE_GATHERV: "1", SGLANG_DP_USE_REDUCE_SCATTER: "1", MORI_MAX_DISPATCH_TOKENS_DECODE: "256"}, args: {tp-size: 8, dp-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, swa-full-tokens-ratio: 0.15, mem-fraction-static: 0.92, max-running-requests: 384, cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48], speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}} + benchmark: {env: {CONC: '192', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'}} +override_c256: + roles: {prefill: {gpus: 8, env: {GPU_MAX_HW_QUEUES: "5", SGLANG_SHARED_EXPERT_TP1: "1", SGLANG_DP_SHARED_EXPERT_LOCAL: "1", SGLANG_DP_USE_GATHERV: "1", SGLANG_DP_USE_REDUCE_SCATTER: "1", }, args: {tp-size: 8, dp-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, swa-full-tokens-ratio: 0.15, mem-fraction-static: 0.92, chunked-prefill-size: 65536, enable-two-batch-overlap: false, max-running-requests: 512, speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}, decode: {gpus: 8, env: {GPU_MAX_HW_QUEUES: "5", SGLANG_SHARED_EXPERT_TP1: "1", SGLANG_DP_SHARED_EXPERT_LOCAL: "1", SGLANG_DP_USE_GATHERV: "1", SGLANG_DP_USE_REDUCE_SCATTER: "1", MORI_MAX_DISPATCH_TOKENS_DECODE: "256"}, args: {tp-size: 8, dp-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, swa-full-tokens-ratio: 0.15, mem-fraction-static: 0.92, max-running-requests: 512, cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64], speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}} + benchmark: {env: {CONC: '256', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'}} diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml index adc2d1d3a3..beb34c24dd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml @@ -170,7 +170,7 @@ frontend: node: first_decode benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: MODEL: deepseek-ai/DeepSeek-V4-Pro MODEL_PREFIX: dsv4 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml index f4f16337f4..27a8c25686 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml @@ -170,7 +170,7 @@ frontend: node: first_decode benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: MODEL: deepseek-ai/DeepSeek-V4-Pro MODEL_PREFIX: dsv4 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml index 08b68b2202..f6f914fd83 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml @@ -170,7 +170,7 @@ frontend: node: first_decode benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: MODEL: deepseek-ai/DeepSeek-V4-Pro MODEL_PREFIX: dsv4 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml index 46be52c30f..1bf613d495 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml @@ -171,7 +171,7 @@ frontend: node: first_decode benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: MODEL: deepseek-ai/DeepSeek-V4-Pro MODEL_PREFIX: dsv4 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml index 373a12989c..818f5da7cc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml @@ -177,7 +177,7 @@ frontend: node: first_decode benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: MODEL: deepseek-ai/DeepSeek-V4-Pro MODEL_PREFIX: dsv4 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml index 6e2e980ebd..7a5e0c4691 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml @@ -193,7 +193,7 @@ frontend: node: first_decode benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: MODEL: deepseek-ai/DeepSeek-V4-Pro MODEL_PREFIX: dsv4 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml index 9c604d726b..49d15b0f32 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml @@ -98,7 +98,7 @@ srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml index 71f279e59b..c34d66a75c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml @@ -146,7 +146,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml index 58e999b60d..d9752e3eef 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml @@ -145,7 +145,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml index 845cbf966f..2fa022b714 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml @@ -108,7 +108,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml index f9c62a6a24..6e1cfc45eb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml @@ -206,7 +206,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml index 01c6e6e5af..4bd6b4ea1d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml @@ -206,7 +206,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml index 178f6af471..97eb57e1a0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml @@ -119,7 +119,7 @@ srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml index 0bc695d779..57ab2522ce 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml @@ -208,7 +208,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml index 9ead5965b0..908e5ad7ef 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml @@ -208,7 +208,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml index fb540cdbf5..63e789d9b9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml @@ -116,7 +116,7 @@ srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml index dda2443295..329626fbcc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml @@ -145,7 +145,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml index f17faead2a..e99921d95c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml @@ -146,7 +146,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml index acdca7c858..96d135c0f7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml @@ -213,7 +213,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml index bb98740ab7..c91da9bb8e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml @@ -213,7 +213,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml index 62867c6566..130997107c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml @@ -206,7 +206,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml index d9d280a6c9..3f91ed3e2d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml @@ -213,7 +213,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml index 922a5343a8..02c44990af 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml @@ -191,7 +191,7 @@ benchmark: # The AgentX client uses localhost:8000, so colocate it with the Dynamo # frontend launched on the first decode node. type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml index 2670488be1..82256d088c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml @@ -7,7 +7,7 @@ name: "svf-vllm-disagg-gb300-1p6d-dep4-tp4-agentic" # the fixed-seq-len 1p6d baseline at the same concurrency point (192). # # Divergence vs the 8k1k sibling: -# - benchmark.type: sa-bench -> custom (hands off to agentic_srt.sh) +# - benchmark.type: sa-bench -> custom (hands off to srt_agentic.sh) # - max-model-len: removed (let vLLM derive from model config; agentic # trajectories blow past any small explicit cap) # - no-enable-prefix-caching: dropped (prefix caching MUST be on for @@ -154,7 +154,7 @@ srun_options: mem: "0" # gb300-nv: pyxis maps the calling user (sa-shared) into the container as # uid 345200007. dpkg refuses to run without EUID 0 even though - # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the agentic_srt + # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the srt_agentic # apt-get install git step fails. --container-remap-root asks pyxis to # remap us to uid 0 inside the container. srt-slurm renders empty-string # values as flag-only srun args (see core/slurm.py:250). @@ -162,7 +162,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml index 2b234884ff..9e35de536e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml @@ -208,7 +208,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml index 7b8b762e6f..2e6ab175fc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml @@ -208,7 +208,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml index 20e836b93b..08766587b8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml @@ -7,7 +7,7 @@ name: "svf-vllm-disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic" # node. Sized for concurrency 4096 with deep_gemm_mega_moe on both workers. # # Divergence vs the 8k1k sibling: -# - benchmark.type: sa-bench -> custom (hands off to agentic_srt.sh) +# - benchmark.type: sa-bench -> custom (hands off to srt_agentic.sh) # - max-model-len: removed (let vLLM derive from model config; agentic # trajectories blow past any small explicit cap) # - no-enable-prefix-caching: dropped (prefix caching MUST be on for @@ -163,7 +163,7 @@ srun_options: mem: "0" # gb300-nv: pyxis maps the calling user (sa-shared) into the container as # uid 345200007. dpkg refuses to run without EUID 0 even though - # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the agentic_srt + # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the srt_agentic # apt-get install git step fails. --container-remap-root asks pyxis to # remap us to uid 0 inside the container. srt-slurm renders empty-string # values as flag-only srun args (see core/slurm.py:250). @@ -171,7 +171,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/tilert/b200-fp8/agentx/disagg-1p1d-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/tilert/b200-fp8/agentx/disagg-1p1d-tp8-mtp.yaml index 1fff29118f..1d31398dc9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/tilert/b200-fp8/agentx/disagg-1p1d-tp8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/tilert/b200-fp8/agentx/disagg-1p1d-tp8-mtp.yaml @@ -87,7 +87,7 @@ benchmark: mkdir -p /infmax-workspace/LOGS/agentic export RESULT_DIR=/infmax-workspace/LOGS/agentic export AGENTIC_OUTPUT_DIR=/infmax-workspace - exec bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + exec bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml index 92cd251488..fe123e03bd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml @@ -1,7 +1,7 @@ # Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 1. # # One TP8 worker serves prefill and decode on a single node. The sweep matrix -# supplies the concurrency list; agentic_srt.sh replays every point against +# supplies the concurrency list; srt_agentic.sh replays every point against # this one server. Acceptance is pinned to the golden thinking-on AL for three # speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). schema: 2 @@ -83,7 +83,7 @@ roles: enable-cache-report: true benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml index d28d641d54..4fdb449e44 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml @@ -1,7 +1,7 @@ # Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 4. # # One TP8 worker serves prefill and decode on a single node. The sweep matrix -# supplies the concurrency list; agentic_srt.sh replays every point against +# supplies the concurrency list; srt_agentic.sh replays every point against # this one server. Acceptance is pinned to the golden thinking-on AL for three # speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). schema: 2 @@ -83,7 +83,7 @@ roles: enable-cache-report: true benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml index 6b89ba241b..6b0cbfa53a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml @@ -1,7 +1,7 @@ # Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 8. # # One TP8 worker serves prefill and decode on a single node. The sweep matrix -# supplies the concurrency list; agentic_srt.sh replays every point against +# supplies the concurrency list; srt_agentic.sh replays every point against # this one server. Acceptance is pinned to the golden thinking-on AL for three # speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). schema: 2 @@ -83,7 +83,7 @@ roles: enable-cache-report: true benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml index dd875b3708..f9b90266de 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml @@ -168,7 +168,7 @@ health_check: interval_seconds: 10 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml index ac814c39a1..505c4935bf 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml @@ -160,7 +160,7 @@ health_check: interval_seconds: 10 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml index c56e03e9be..e0a6fcaec6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml @@ -97,7 +97,7 @@ health_check: interval_seconds: 10 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml index d845ec11e4..c192760a10 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml @@ -97,7 +97,7 @@ health_check: interval_seconds: 10 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml index 6b45a9c09e..98a00e015f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml @@ -97,7 +97,7 @@ health_check: interval_seconds: 10 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml index 2be342ac98..3f2650c0f7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml @@ -115,7 +115,7 @@ telemetry: benchmark: type: custom concurrencies: [1] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml index 27721263a6..dad9b18d2b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml @@ -168,7 +168,7 @@ health_check: interval_seconds: 10 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml index 51f260e38c..13b242d780 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml @@ -168,7 +168,7 @@ health_check: interval_seconds: 10 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml index dcb66f55d2..a586a319ea 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml @@ -176,7 +176,7 @@ health_check: interval_seconds: 10 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml index 9bf160a4c8..ecf94a8d60 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml @@ -182,7 +182,7 @@ base: interval_seconds: 10 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg.yaml index 6a70bf63e3..72d65baae7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg.yaml @@ -4,7 +4,7 @@ # Flat single-variant schema the agentic CI flow expects: applied via # CONFIG_FILE= with no zip_override selector, so it must NOT be # base:-wrapped. Concurrency is not a recipe field; the GHA matrix fans out -# one job per concurrency from the master-config conc-list into agentic_srt.sh. +# one job per concurrency from the master-config conc-list into srt_agentic.sh. schema: 2 name: gb300-fp4-glm5.2-agentx-agg model: @@ -97,7 +97,7 @@ health_check: interval_seconds: 10 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml index e9d5b35e34..36ef2c32ff 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml @@ -154,7 +154,7 @@ base: interval_seconds: 10 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c12.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c12.yaml index 6595f70ae7..70c86fd033 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c12.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c12.yaml @@ -118,7 +118,7 @@ telemetry: benchmark: type: custom use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c16.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c16.yaml index 682d82606d..c3ee74a2e9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c16.yaml @@ -118,7 +118,7 @@ telemetry: benchmark: type: custom use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c2.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c2.yaml index a96f5d480a..3d2b8a423d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c2.yaml @@ -118,7 +118,7 @@ telemetry: benchmark: type: custom use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c4.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c4.yaml index 22d878c993..60f9736aa1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c4.yaml @@ -118,7 +118,7 @@ telemetry: benchmark: type: custom use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c8.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c8.yaml index 4a9e502c78..a4a603a264 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c8.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c8.yaml @@ -118,7 +118,7 @@ telemetry: benchmark: type: custom use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml index 974b066c92..c8ace70283 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml @@ -152,14 +152,13 @@ telemetry: required: true startup_timeout_seconds: 120 request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 dcgm_exporter: container_image: dcgm-exporter port: 9401 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml index c234617ca4..f50ceb85af 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml @@ -159,7 +159,7 @@ benchmark: type: custom # Replaced with CONC_LIST by the launcher before submission. concurrencies: [1] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml index 2d0380aec2..7ea49d9257 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml @@ -166,7 +166,7 @@ frontend: node: head benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml index dcf151af95..efa7b8fb86 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml @@ -164,7 +164,7 @@ frontend: node: head benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml index 9408847d11..8343835cc1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml @@ -166,7 +166,7 @@ frontend: node: head benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml index 507a1c08eb..70947a0180 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml @@ -166,7 +166,7 @@ frontend: node: head benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml index bb733a601e..60a66c1a5b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml @@ -167,7 +167,7 @@ frontend: node: head benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml index 7a9647373a..e0cbc51b2e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml @@ -167,7 +167,7 @@ frontend: node: head benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml index 532de487ee..ad4390d7e0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml @@ -127,7 +127,7 @@ telemetry: benchmark: type: custom concurrencies: [1] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml index 590cfc7f7e..fed72d219b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml @@ -127,7 +127,7 @@ telemetry: benchmark: type: custom concurrencies: [14] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml index cdcb3fb8d0..ccd6894f55 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml @@ -127,7 +127,7 @@ telemetry: benchmark: type: custom concurrencies: [24] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml index a45f184bec..5dd98d53a6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml @@ -127,7 +127,7 @@ telemetry: benchmark: type: custom concurrencies: [4] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml index a8d44175ac..250c4a5b35 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml @@ -127,7 +127,7 @@ telemetry: benchmark: type: custom concurrencies: [48] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml index 2bff214dd2..c2cffc57b4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml @@ -127,7 +127,7 @@ telemetry: benchmark: type: custom concurrencies: [8] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml index 28018a5910..b9d2212953 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml @@ -126,7 +126,7 @@ telemetry: benchmark: type: custom concurrencies: [96] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml index ceb0efb874..fa1ef92c37 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml @@ -159,7 +159,7 @@ benchmark: type: custom client_placement: head concurrencies: [1, 2, 4, 8, 16] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml index 81af8a3005..ed5eef9320 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml @@ -155,7 +155,7 @@ benchmark: type: custom client_placement: head concurrencies: [8, 40, 48] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml index ff19c7103b..9d8b740e1f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml @@ -148,7 +148,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml index 4edaabdfc5..8bfda2949e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml @@ -145,7 +145,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml index aace6df184..58bae20c35 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml @@ -137,7 +137,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml index a838dc8f73..9c597e1701 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml @@ -134,7 +134,7 @@ srun_options: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml index 33f3115102..8346ab2963 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml @@ -128,7 +128,7 @@ benchmark: type: custom client_placement: head concurrencies: [16] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml index cf98995eec..a6cfad55f0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml @@ -128,7 +128,7 @@ benchmark: type: custom client_placement: head concurrencies: [32] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml index 57498c8969..f86e4d3119 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml @@ -127,7 +127,7 @@ benchmark: type: custom client_placement: head concurrencies: [48] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml index c7e1da7937..7ae1438a5d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml @@ -127,7 +127,7 @@ benchmark: type: custom client_placement: head concurrencies: [72] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml index 2bb53eaee4..930d9425d0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml @@ -127,7 +127,7 @@ benchmark: type: custom client_placement: head concurrencies: [96] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml index bac87abe62..a15277f843 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml @@ -145,7 +145,7 @@ telemetry: benchmark: type: custom concurrencies: [48, 52, 56] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml index 113a947f37..a921e677a0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml @@ -143,7 +143,7 @@ telemetry: benchmark: type: custom concurrencies: [1, 4] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml index dd9a30f069..0c77c8cb70 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml @@ -174,8 +174,7 @@ srun_options: telemetry: enabled: true - provider: dcgm-power - default_frequency: 1.0 + collect_interval_ms: 1000 storage_subdir: power required: true startup_timeout_seconds: 120 @@ -189,7 +188,7 @@ benchmark: type: custom client_placement: head concurrencies: [48, 52, 56] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml index 6ea49208c6..d640ad21d2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml @@ -174,8 +174,7 @@ srun_options: telemetry: enabled: true - provider: dcgm-power - default_frequency: 1.0 + collect_interval_ms: 1000 storage_subdir: power required: true startup_timeout_seconds: 120 @@ -189,7 +188,7 @@ benchmark: type: custom client_placement: head concurrencies: [32, 48, 64] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml index 4bf4ecc656..9565c64c8e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml @@ -174,8 +174,7 @@ srun_options: telemetry: enabled: true - provider: dcgm-power - default_frequency: 1.0 + collect_interval_ms: 1000 storage_subdir: power required: true startup_timeout_seconds: 120 @@ -189,7 +188,7 @@ benchmark: type: custom client_placement: head concurrencies: [32, 48] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml index 0cd2a16883..92dbd01512 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml @@ -174,8 +174,7 @@ srun_options: telemetry: enabled: true - provider: dcgm-power - default_frequency: 1.0 + collect_interval_ms: 1000 storage_subdir: power required: true startup_timeout_seconds: 120 @@ -189,7 +188,7 @@ benchmark: type: custom client_placement: head concurrencies: [1] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml index 0ae9ca0fe6..d5b84a386d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml @@ -97,7 +97,7 @@ telemetry: benchmark: type: custom concurrencies: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml index 6c241ec561..ed0ce03355 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml @@ -85,8 +85,7 @@ srun_options: telemetry: enabled: true - provider: dcgm-power - default_frequency: 1.0 + collect_interval_ms: 1000 storage_subdir: power required: true startup_timeout_seconds: 120 @@ -99,7 +98,7 @@ telemetry: benchmark: type: custom concurrencies: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml index edfb3d9963..249925f675 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml @@ -86,8 +86,7 @@ srun_options: telemetry: enabled: true - provider: dcgm-power - default_frequency: 1.0 + collect_interval_ms: 1000 storage_subdir: power required: true startup_timeout_seconds: 120 @@ -100,7 +99,7 @@ telemetry: benchmark: type: custom concurrencies: [8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml index c9574016ba..14d4fcbe7f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml @@ -124,7 +124,7 @@ frontend: no-kv-events: true benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml index cef285921e..f18cbf28d0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml @@ -129,7 +129,7 @@ frontend: no-kv-events: true benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml index 55c9e38641..4bbe9c29e0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml @@ -134,7 +134,7 @@ frontend: no-kv-events: true benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml index 25e2a5df3e..d0c79d1e41 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml @@ -134,7 +134,7 @@ frontend: no-kv-events: true benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml index b6315eaa71..5cecb444a3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml @@ -134,7 +134,7 @@ frontend: no-kv-events: true benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml index 1d29eebbc1..bf0920ce90 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml @@ -138,7 +138,7 @@ frontend: no-kv-events: true benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml index 9279351de5..1eaa2bce7b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml @@ -119,7 +119,7 @@ frontend: no-kv-events: true benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4-vllm-simple.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4-vllm-simple.yaml index eb661f2c60..1ec16d4f66 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4-vllm-simple.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4-vllm-simple.yaml @@ -92,7 +92,7 @@ sbatch_directives: {cpus-per-task: "144", mem: "0"} srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4.yaml index 33324c4699..8bb79b49b4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4.yaml @@ -90,7 +90,7 @@ sbatch_directives: {cpus-per-task: "144", mem: "0"} srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep8.yaml index 020dd3d4d0..49fda02e0c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep8.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep8.yaml @@ -90,7 +90,7 @@ sbatch_directives: {cpus-per-task: "144", mem: "0"} srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml index 5b4a692ed4..6e8399d239 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml @@ -95,7 +95,7 @@ srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml index cd56742486..a36e5ff622 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml @@ -92,7 +92,7 @@ srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple.yaml index f88160a9cf..54390af54e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple.yaml @@ -97,7 +97,7 @@ srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4.yaml index f8fdb3c152..e956a39828 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4.yaml @@ -100,7 +100,7 @@ srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml index 590eac5eaf..822ca199c8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml @@ -95,7 +95,7 @@ srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep4.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep4.yaml index d04d2c6299..d7d6023eaf 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep4.yaml @@ -126,7 +126,7 @@ sbatch_directives: {cpus-per-task: "144", mem: "0"} srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml index ea049b7320..a9f4cbc3eb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml @@ -155,7 +155,7 @@ srun_options: {container-remap-root: ""} benchmark: type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml index 2a0121782d..2aa021e119 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml @@ -162,7 +162,7 @@ srun_options: {container-remap-root: ""} benchmark: type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml index 513b7a7954..dd50b12a6f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml @@ -155,7 +155,7 @@ srun_options: {container-remap-root: ""} benchmark: type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml index 44e9395766..02b9c15d97 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml @@ -165,7 +165,7 @@ srun_options: benchmark: type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml index d121a66c8b..3cac730921 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml @@ -165,7 +165,7 @@ srun_options: benchmark: type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml index 88bc24e782..8faf2a5448 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml @@ -165,7 +165,7 @@ srun_options: benchmark: type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml index 3677f72f71..cf4b418c39 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml @@ -165,7 +165,7 @@ srun_options: benchmark: type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml index 1e6fadc381..400798973b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml @@ -168,7 +168,7 @@ srun_options: benchmark: type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml index 7efc02a2ab..e541182719 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml @@ -168,7 +168,7 @@ srun_options: benchmark: type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml index d0e7aa0acf..6cf7e59d1e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml @@ -165,7 +165,7 @@ srun_options: benchmark: type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml index 142bad0347..cc193dd0c8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml @@ -165,7 +165,7 @@ srun_options: benchmark: type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml index 2ca720c02e..66e6e7135d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml @@ -165,7 +165,7 @@ srun_options: benchmark: type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml index f980e5f3fa..a361fd07c9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml @@ -165,7 +165,7 @@ srun_options: benchmark: type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml index a3f8bb681c..61baf8076f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml @@ -154,7 +154,7 @@ telemetry: port: 9401 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh concurrencies: - 16 env: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml index a9af6f1251..0744261c8d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml @@ -154,7 +154,7 @@ telemetry: port: 9401 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh concurrencies: - 24 env: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml index 5dab4917a0..3fcbfe1ba6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml @@ -154,7 +154,7 @@ telemetry: port: 9401 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh concurrencies: - 32 env: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml index 5197d3be5a..96ce0b05f2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml @@ -155,7 +155,7 @@ telemetry: port: 9401 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh concurrencies: - 48 env: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml index ac6a00a1d8..816a5bdf42 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml @@ -155,7 +155,7 @@ telemetry: port: 9401 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh concurrencies: - 64 env: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml index 2492edcae4..11f0f770cb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml @@ -154,7 +154,7 @@ telemetry: port: 9401 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh concurrencies: - 8 env: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml index c73b3f0f1f..22cf32d1b7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml @@ -162,7 +162,7 @@ sbatch_directives: cpus-per-task: '144' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml index 0150c174ba..494ae4b77c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml @@ -162,7 +162,7 @@ sbatch_directives: cpus-per-task: '144' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml index cecb894de7..ba725e8464 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml @@ -162,7 +162,7 @@ sbatch_directives: cpus-per-task: '144' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml index d5b99fbbcd..0b66b22cf3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml @@ -162,7 +162,7 @@ sbatch_directives: cpus-per-task: '144' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache.yaml index f007aad45b..2b48d3b7fc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache.yaml @@ -163,7 +163,7 @@ sbatch_directives: cpus-per-task: '144' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml index a8d1f5bc62..3b0e6a8083 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml @@ -162,7 +162,7 @@ sbatch_directives: cpus-per-task: '144' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml index 1b1fe769ad..69912c3f58 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml @@ -162,7 +162,7 @@ sbatch_directives: cpus-per-task: '144' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml index d88c06b458..3273e19273 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml @@ -162,7 +162,7 @@ sbatch_directives: cpus-per-task: '144' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml index ce87f0369b..122687183c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml @@ -162,7 +162,7 @@ sbatch_directives: cpus-per-task: '144' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml index 6912536759..fe54b630a5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml @@ -99,7 +99,7 @@ sbatch_directives: {mem: "0", cpus-per-task: "144"} srun_options: {mem: "0", container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml index 624e4e7f68..919890bc13 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml @@ -100,7 +100,7 @@ sbatch_directives: {mem: "0", cpus-per-task: "144"} srun_options: {mem: "0", container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml index 35991d1a61..baa7b9e34f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml @@ -101,7 +101,7 @@ sbatch_directives: {mem: "0", cpus-per-task: "144"} srun_options: {mem: "0", container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-hicache.yaml index eab7f2107d..c0077e00fb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-hicache.yaml @@ -73,7 +73,7 @@ sbatch_directives: {mem: "0", cpus-per-task: "144"} srun_options: {mem: "0", container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml index db3e9116f2..81f1bae1b7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml @@ -95,7 +95,7 @@ srun_options: {mem: "0", container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml index 4697c70678..35abe029cc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml @@ -98,7 +98,7 @@ srun_options: {mem: "0", container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml index ea6ea024e3..bf14c75b64 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml @@ -95,7 +95,7 @@ srun_options: {mem: "0", container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml index 1e6f6493fe..4d71db73f4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml @@ -72,7 +72,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml index b6579ff4ac..cb64f23657 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml @@ -72,7 +72,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml index 0f5a27a447..2271ef3e29 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml @@ -72,7 +72,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml index 9b3d398808..3335faf1b3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml @@ -72,7 +72,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml index 7a5a8417a6..1ab4c4a73d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml @@ -117,7 +117,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml index ad5b4c86e7..b3dd048818 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml @@ -120,7 +120,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml index 755cc77d0e..b3d34533f4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml @@ -72,7 +72,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml index 24a388391d..cbd83c1850 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml @@ -72,7 +72,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml index ee3999af7b..4a6f68d048 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml @@ -72,7 +72,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml index bc672a8342..cb75cb22f3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml @@ -117,7 +117,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml index 3580fbbe29..e5d160a211 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml @@ -197,7 +197,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml index 908f37b284..3c6ffd223b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml @@ -192,7 +192,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml index a9e4f40a9a..cfc162993a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml @@ -192,7 +192,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml index 3854d0ad20..58a83e4df2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml @@ -192,7 +192,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml index d8325d5792..6645e6c1ee 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml @@ -192,7 +192,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml index 8a776aa9af..67d10772d9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml @@ -192,7 +192,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml index c0ff95a0f7..b5973a79e0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml @@ -192,7 +192,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml index e790701576..96fc807eaa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml @@ -236,7 +236,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml index 9146587211..9f8f0e1b76 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml @@ -236,7 +236,7 @@ srun_options: container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml index f498c1d931..5f2df55f53 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml @@ -207,7 +207,7 @@ frontend: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml index 076f17ae91..ae71196e1e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml @@ -204,7 +204,7 @@ frontend: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml index 091438994f..6c0b923ce1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml @@ -206,7 +206,7 @@ frontend: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml index f3134246e6..5db9ff2de7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml @@ -221,7 +221,7 @@ frontend: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml index cddc0d8a81..559ab778a5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml @@ -205,7 +205,7 @@ frontend: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml index 3c0b23c7b6..e9e81c54fc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml @@ -208,7 +208,7 @@ frontend: benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..2eb00ff3f9 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,76 @@ +# DeepSeek-V4.1-Flash AgentX on B200 with SGLang native DSpark. The KV cache +# is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-b200-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + # Keep active decode requests progressing while long prefixes are queued. + prefill-decode-interval: 16 + # DSpark is the checkpoint's bundled draft; the block size is its only knob. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + # Admission never exceeds the captured decode graph batch. + cuda-graph-max-bs-decode: 64 + reasoning-parser: auto + tool-call-parser: auto + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV41_REASONING_EFFORT: high + # Row-sharded host Engram tables free HBM for the 1M-context working set. + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC, capped at the graph batch. SWA +# prefix tails scale with CONC (TP4 64x, TP2 128x) within [128, 4096]. TP2 +# needs a larger static fraction and a smaller prefill chunk to fit, and +# expandable segments against fragmentation. Saturation points get a longer +# warmup drain. +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 128, max-running-requests: 2}, }}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 128, max-running-requests: 4}, }}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 256, max-running-requests: 8}, }}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 512, max-running-requests: 16}, }}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 1024, max-running-requests: 32}, }}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 2048, max-running-requests: 64}, }}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 4096, max-running-requests: 64}, }}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 4096, max-running-requests: 64}, }}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 128, max-running-requests: 2}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}, }}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 256, max-running-requests: 4}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}, }}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 512, max-running-requests: 8}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}, }}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 1024, max-running-requests: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}, }}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..eccbd16327 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml @@ -0,0 +1,101 @@ +# DeepSeek-V4.1-Flash AgentX on H200 with SGLang native DSpark, following the +# published vLLM baseline topologies TP4/EP1 and TP8/EP1. The KV cache is +# GPU-resident. https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 +base: + schema: 2 + name: dsv41flash-fp4-h200-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17 + precision: fp4 + resources: + gpu_type: h200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + # Measured H200 launch configurations: the replicated projection on TP4 and + # TP8, plus the qualified TP8 sharded shapes. + setup_script: dsv41flash-h200-block32.sh + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + data-parallel-size: 1 + expert-parallel-size: 1 + attention-backend: dsv4 + # Native Marlin pads the TP-sharded 2304-wide experts (TP4=576, TP8=288) + # that the SM90 CUTLASS method rejects; both consume the MXFP4 weights. + moe-runner-backend: marlin + # Below the cookbook's 0.8, with a bounded prefill chunk: the sparse + # attention indexer and DSpark prefill buffers scale with chunk times the + # 1M context. The default 16384 chunk exhausted HBM on 66k-99k-token + # prompts, and 8192 OOMed c32 (run 35308550355). + mem-fraction-static: 0.70 + chunked-prefill-size: 4096 + # Keep active decode requests progressing while long prefixes are queued. + prefill-decode-interval: 16 + enable-decoder-swa-bounded-replay: true + # The default 4*max-running-requests retained too few SWA prefix tails: + # C16 exhausted its 94,976-slot SWA pool while the full pool had room. + swa-prefix-tails: 1024 + # DSpark is the checkpoint's bundled draft; the block size is its only knob. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + # A verify step above the captured batch runs eagerly and allocates its + # attention workspace, which OOMed the eval at 128 running requests + # (run 35306704553), so admission never exceeds the graph batch. + cuda-graph-max-bs-decode: 64 + reasoning-parser: auto + tool-call-parser: auto + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV41_REASONING_EFFORT: high + # Move the fp8 Engram tables to row-sharded host memory, freeing ~23 GiB + # of HBM per GPU. With them on GPU, c128 died at startup (run 35304458924). + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC, capped at the decode graph batch; +# saturation points carry a larger in-flight working set than the 30-minute +# warmup drain allows. +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 2}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 4}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 8}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 16}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 32}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 64}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 64}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 64}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 2}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp8_c2: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 4}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 8}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp8_c8: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 16}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 32}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp8_c32: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 64}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp8_c64: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 64}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp8_c128: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 64}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..366f501006 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml @@ -0,0 +1,77 @@ +# Qwen3.5 FP8 AgentX on H200 with SGLang EAGLE MTP; attention is flashinfer +# (trtllm_mha is Blackwell-only). +base: + schema: 2 + name: qwen3.5-fp8-h200-sglang-agentic + model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9 + precision: fp8 + resources: + gpu_type: h200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: flashinfer + enable-flashinfer-allreduce-fusion: true + mem-fraction-static: 0.8 + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + enable-metrics: true + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + env: + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_SPEC_V2: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + # 256k-capped with-subagents corpus: the unfiltered corpus has requests up + # to ~1M tokens, beyond this model's TP8 context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_with_subagents_256k + +override_gpu_kv: + benchmark: + env: + KV_OFFLOADING: none +# HiCache extends RadixAttention. Hybrid GDN/Mamba allocates one KV and one +# Mamba host pool per rank: 1234 GB / TP8 / 2 pools = 77 GB each. +override_hicache: + roles: + agg: + args: + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 77 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1234' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml deleted file mode 100644 index 244b8d64dc..0000000000 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml +++ /dev/null @@ -1,75 +0,0 @@ -base: - schema: 2 - name: qwen3.5-fp4-mi355x-sglang-mtp-8k1k - model: - path: hf:amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 - container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913 - precision: fp4 - resources: - gpu_type: mi355x - gpus_per_node: 8 - frontend: - type: sglang - enable_multiple_frontends: false - observability: - enabled: false - tachometer: - enabled: false - engine: sglang - roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - args: - trust-remote-code: true - tensor-parallel-size: 2 - attention-backend: aiter - mem-fraction-static: 0.8 - model-loader-extra-config: '{"enable_multithread_load": true}' - watchdog-timeout: 1200 - disable-radix-cache: true - max-running-requests: 4 - page-size: 16 - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - kv-cache-dtype: fp8_e4m3 - data-parallel-size: 1 - expert-parallel-size: 1 - served-model-name: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 - enable-metrics: false - env: - SGLANG_USE_AITER: '1' - SGLANG_USE_AITER_UNIFIED_ATTN: '1' - AITER_FLYDSL_FORCE: '1' - SGLANG_MAMBA_SSM_DTYPE: bfloat16 - ROCM_QUICK_REDUCE_QUANTIZATION: INT8 - benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/single_node/srt_fixed_sequence.sh - env: - MODEL: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 - ISL: '8192' - OSL: '1024' - RANDOM_RANGE_RATIO: '0.8' - USE_CHAT_TEMPLATE: 'true' -zip_override_tp2_ep1: - roles: - agg: - args: - max-running-requests: [4, 8, 16, 32, 64, 128] - benchmark: - env: - CONC: ['4', '8', '16', '32', '64', '128'] -zip_override_tp4_ep1: - roles: - agg: - gpus: 4 - args: - max-running-requests: [4, 8, 16] - tensor-parallel-size: 4 - benchmark: - env: - CONC: ['4', '8', '16'] diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..76e92bfd7e --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml @@ -0,0 +1,61 @@ +# Qwen3.8-Flash-Next FP8 AgentX on H200 via SGLang with native NEXTN MTP. +base: + schema: 2 + name: qwen3.8next-fp8-h200-sglang-agentic + model: + path: hf:Qwen/Qwen3.8-Flash-Next-FP8 + container: lmsysorg/sglang:qwen38flashnext + precision: fp8 + resources: + gpu_type: h200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + args: + served-model-name: Qwen/Qwen3.8-Flash-Next-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 4 + data-parallel-size: 1 + mem-fraction-static: 0.85 + chunked-prefill-size: 8192 + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + mamba-ssm-dtype: float32 + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + reasoning-parser: auto + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_SPEC_V2: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.8-Flash-Next-FP8 + # 256k-capped with-subagents corpus, within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_with_subagents_256k + +# One variant per point: admission is 2x CONC and the graph batch is CONC. +override_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1'}}} +override_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4'}}} +override_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8'}}} +override_c12: {roles: {agg: {args: {max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12'}}} +override_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '16'}}} diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/srt_agentic.sh similarity index 76% rename from benchmarks/multi_node/agentic_srt.sh rename to benchmarks/srt_agentic.sh index 338fcb966b..c0f61583cb 100644 --- a/benchmarks/multi_node/agentic_srt.sh +++ b/benchmarks/srt_agentic.sh @@ -2,12 +2,20 @@ set -eo pipefail set -x -# Client-only agentic trace replay for srt-slurm multinode jobs. +# Client-only AgentX trace replay for single- and multi-node srt-slurm jobs. # srt-slurm owns server startup; this script runs as benchmark.type=custom -# against the already-ready frontend on the head node. +# against the already-ready frontend. Multi-node batches replay each CONC_LIST +# point in turn; a single-node job replays its one CONC point. -source "$(dirname "${BASH_SOURCE[0]}")/../benchmark_lib.sh" --validation-only -check_env_vars INFMAX_CONTAINER_WORKSPACE RESULT_DIR EVAL_ONLY AIPERF_DRAIN_TIMEOUT_SECONDS AIPERF_DRAIN_POLL_SECONDS +# Jobs inherit the legacy scripts' /workspace, which srt-slurm does not mount; +# fall back to the repo mount this client runs from. +if [[ ! -f "${INFMAX_CONTAINER_WORKSPACE:-}/benchmarks/benchmark_lib.sh" ]]; then + INFMAX_CONTAINER_WORKSPACE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +fi +: "${IS_MULTINODE:=false}" "${PORT:=8000}" +export INFMAX_CONTAINER_WORKSPACE IS_MULTINODE PORT +source "$INFMAX_CONTAINER_WORKSPACE/benchmarks/benchmark_lib.sh" --validation-only +check_env_vars RESULT_DIR EVAL_ONLY source "$INFMAX_CONTAINER_WORKSPACE/benchmarks/benchmark_lib.sh" if [[ -n "${SRT_FRONTEND_HOST:-}" ]]; then @@ -41,9 +49,21 @@ if [[ -z "${AIPERF_SERVER_URL:-}" ]]; then fi echo "Using srt-slurm frontend endpoint: $AIPERF_SERVER_URL" +# A router frontend does not re-export engine metrics; read them from each worker. +if [[ -z "${AIPERF_SERVER_METRICS_URLS:-}" && "${SRTCTL_FRONTEND_TYPE:-}" != dynamo ]]; then + endpoints="${SRT_AGG_ENDPOINTS:-${SRT_PREFILL_ENDPOINTS:+$SRT_PREFILL_ENDPOINTS,}${SRT_DECODE_ENDPOINTS:-}}" + if [[ -n "${endpoints%,}" ]]; then + AIPERF_SERVER_METRICS_URLS=$(sed -E 's#([^,]+)#http://\1/metrics#g' <<< "${endpoints%,}") + export AIPERF_SERVER_METRICS_URLS + fi +fi + BASE_RESULT_DIR="${RESULT_DIR}" BASE_RESULT_FILENAME="$RESULT_FILENAME" read -r -a CONCURRENCIES <<< "${CONC_LIST:-$CONC}" +if (( ${#CONCURRENCIES[@]} > 1 )); then + check_env_vars AIPERF_DRAIN_TIMEOUT_SECONDS AIPERF_DRAIN_POLL_SECONDS +fi if [ "${#CONCURRENCIES[@]}" -eq 0 ]; then echo "ERROR: CONC_LIST must contain at least one concurrency" >&2 @@ -138,8 +158,11 @@ PY for index in "${!CONCURRENCIES[@]}"; do concurrency="${CONCURRENCIES[$index]}" export CONC="$concurrency" - export RESULT_FILENAME="${BASE_RESULT_FILENAME}_conc${concurrency}" - RESULT_DIR="${BASE_RESULT_DIR}/conc_${concurrency}" + # Multi-node collection expects per-point names; a single-node job keeps the workflow's. + if [[ -n "${CONC_LIST:-}" ]]; then + export RESULT_FILENAME="${BASE_RESULT_FILENAME}_conc${concurrency}" + RESULT_DIR="${BASE_RESULT_DIR}/conc_${concurrency}" + fi mkdir -p "$RESULT_DIR" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 499a5c11b9..89cd673dac 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1123,9 +1123,10 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - # All arms use umbp-linker. DSpark gamma 6 at c4/c16 (golden AL peak), 3 above. + # All arms offload to umbp-linker. The recipe carries each point's sizing and + # DSpark draft length (6 at c4/c16, 3 above). - spec-decoding: "draft_model" - conc-list: [ 4 ] + conc-list: [4] kv-offloading: dram kv-offload-backend: { name: umbp-linker } prefill: @@ -1134,18 +1135,14 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907" + - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c4" decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=6" - spec-decoding: "draft_model" - conc-list: [ 16 ] + conc-list: [16] kv-offloading: dram kv-offload-backend: { name: umbp-linker } prefill: @@ -1154,18 +1151,14 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907" + - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c16" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=6" - spec-decoding: "draft_model" - conc-list: [ 32, 48 ] + conc-list: [32] kv-offloading: dram kv-offload-backend: { name: umbp-linker } prefill: @@ -1174,18 +1167,30 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907" + - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c32" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false + - spec-decoding: "draft_model" + conc-list: [48] + kv-offloading: dram + kv-offload-backend: { name: umbp-linker } + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" + - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c48" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false - spec-decoding: "draft_model" - conc-list: [ 192, 256 ] + conc-list: [192] kv-offloading: dram kv-offload-backend: { name: umbp-linker } prefill: @@ -1194,17 +1199,28 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: ep: 1 dp-attn: true additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_ROUTER_POLICY=consistent_hashing" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907" + - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c192" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: true + - spec-decoding: "draft_model" + conc-list: [256] + kv-offloading: dram + kv-offload-backend: { name: umbp-linker } + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" + - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c256" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true minimaxm3-fp4-mi355x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:nightly-2a02f6efe319c885e3ccbcecde402e0028f9ec1e diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 7ee2ca0bee..d3908eca37 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5240,8 +5240,8 @@ qwen3.5-fp8-h200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 8, 12, 16] } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml } # Qwen3.8-Flash-Next FP8 AgentX on H200 via SGLang with native NEXTN MTP. # Day-zero recipe. H200 is Hopper, so this arm is FP8: NVFP4 is greyed out for @@ -5260,7 +5260,7 @@ qwen3.8next-fp8-h200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } + - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml } # H200 AgentX MTP frontier with DRAM HiCache. This is intentionally an MTP-only # submission; the model's non-speculative AgentX arm is not included. qwen3.5-fp8-h200-sglang-agentic-hicache-mtp: @@ -8469,8 +8469,8 @@ dsv41flash-fp4-b200-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8] } + - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml } dsv41flash-fp4-gb300-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-af1c01499b289be555c475669ba50a88e96d846e @@ -8544,8 +8544,8 @@ dsv41flash-fp4-h200-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml } dsv4-fp4-b200-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index 06c9ef6cf9..3fbd3adf07 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -191,7 +191,7 @@ Retain `meta_env.json`, `results*.json`, and `sample*.jsonl`. Agentic SWE-bench AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L2104-L2128)). -For multi-node srt-slurm jobs, the benchmark client may run on a different host from the frontend. `agentic_srt.sh` uses an explicit `AIPERF_SERVER_URL` when supplied, otherwise derives it from `SRT_FRONTEND_HOST` and `SRT_FRONTEND_PORT`, and falls back to `localhost:$PORT` only when no remote endpoint is available. Trace replay and inter-point drain checks must use that same resolved endpoint. +For multi-node srt-slurm jobs, the benchmark client may run on a different host from the frontend. `srt_agentic.sh` uses an explicit `AIPERF_SERVER_URL` when supplied, otherwise derives it from `SRT_FRONTEND_HOST` and `SRT_FRONTEND_PORT`, and falls back to `localhost:$PORT` only when no remote endpoint is available. Trace replay and inter-point drain checks must use that same resolved endpoint. Keep non-index engine or router wheels reproducible and immutable: check in the source patch and builder beside the launcher, verify the upstream wheel's digest before patching, assign an explicit local version, and install the published artifact through an exact URL with a SHA256 fragment. A local backport must not use an unreleased upstream version number. diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index 03c0580439..2105ba5d89 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -189,7 +189,7 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \ AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[fast replay 设置](../benchmarks/benchmark_lib.sh#L2104-L2128)）。 -对于多节点 srt-slurm 作业，benchmark client 与 frontend 可能运行在不同主机上。`agentic_srt.sh` 会优先使用显式提供的 `AIPERF_SERVER_URL`；否则从 `SRT_FRONTEND_HOST` 和 `SRT_FRONTEND_PORT` 推导地址；仅在没有远端 endpoint 时回退到 `localhost:$PORT`。Trace replay 和并发点之间的 drain 检查必须使用同一个解析后的 endpoint。 +对于多节点 srt-slurm 作业，benchmark client 与 frontend 可能运行在不同主机上。`srt_agentic.sh` 会优先使用显式提供的 `AIPERF_SERVER_URL`；否则从 `SRT_FRONTEND_HOST` 和 `SRT_FRONTEND_PORT` 推导地址；仅在没有远端 endpoint 时回退到 `localhost:$PORT`。Trace replay 和并发点之间的 drain 检查必须使用同一个解析后的 endpoint。 对于未发布到 package index 的 engine 或 router wheel，必须保证构建可复现且 artifact 不可变：在 launcher 旁签入源码 patch 与构建器，打 patch 前校验上游 wheel 的 digest，分配明确的 local version，并通过带 SHA256 fragment 的精确 URL 安装已发布 artifact。本地 backport 不得冒用尚未发布的上游版本号。 diff --git a/infx/matrix/generate.py b/infx/matrix/generate.py index af7ec486fd..b4fc61deb1 100644 --- a/infx/matrix/generate.py +++ b/infx/matrix/generate.py @@ -968,6 +968,8 @@ def _agentic_entries( Fields.CONC.value: conc, } ) + if benchmark.get(Fields.SRT_RECIPE.value) is not None: + entry[Fields.SRT_RECIPE.value] = benchmark[Fields.SRT_RECIPE.value] exp_name = ( f"{model_code}_tp{tp}_conc{conc}_" f"{agentic_kv_offload_suffix(kv_offloading, kv_offload_backend)}" diff --git a/infx/matrix/validation.py b/infx/matrix/validation.py index 7f774570b9..bf1af5f272 100644 --- a/infx/matrix/validation.py +++ b/infx/matrix/validation.py @@ -309,6 +309,7 @@ class SingleNodeAgenticMatrixEntry(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) image: str + srt_recipe: str | None = Field(default=None, alias=Fields.SRT_RECIPE.value, min_length=1) model: str model_prefix: str = Field(alias=Fields.MODEL_PREFIX.value) precision: str @@ -628,6 +629,7 @@ class AgenticCodingSearchSpaceEntry(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) tp: int | None = None + srt_recipe: str | None = Field(default=None, alias=Fields.SRT_RECIPE.value, min_length=1) pp: int = Field(default=1, gt=0, strict=True) dcp_size: int = Field(default=1, alias=Fields.DCP_SIZE.value, gt=0, strict=True) pcp_size: int = Field(default=1, alias=Fields.PCP_SIZE.value, gt=0, strict=True) diff --git a/infx/srt_slurm/single_node.py b/infx/srt_slurm/single_node.py index 31655f6dc7..a4a5a14fef 100644 --- a/infx/srt_slurm/single_node.py +++ b/infx/srt_slurm/single_node.py @@ -1,4 +1,4 @@ -"""Bind a native single-node SRT recipe to one fixed-sequence matrix point.""" +"""Bind a native single-node SRT recipe to one fixed-sequence or AgentX matrix point.""" from __future__ import annotations @@ -80,9 +80,10 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N if environment["FRAMEWORK"] not in {"sglang", "trt", "atom"}: raise ValueError(f"Unsupported single-node framework: {environment['FRAMEWORK']!r}") spec = spec_parameters(role, engine) - if spec and spec["method"] not in {"eagle", "nextn", "mtp"}: - raise ValueError("Single-node SRT supports only native MTP or no speculation") + if spec and spec["method"] not in {"eagle", "nextn", "mtp", "dspark"}: + raise ValueError("Single-node SRT supports only native MTP, DSpark or no speculation") speculation = "mtp" if spec else "none" + agentic = environment["IS_AGENTIC"] == "1" expected = { "engine": (engine, SINGLE_NODE_ENGINES[environment["FRAMEWORK"]]), "model": (recipe["model"]["path"], f"hf:{environment['MODEL']}"), @@ -96,22 +97,20 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N "benchmark type": (benchmark["type"], "custom"), "benchmark MODEL": (workload["MODEL"], environment["MODEL"]), "SPEC_DECODING": (speculation, environment["SPEC_DECODING"]), - "USE_CHAT_TEMPLATE": (workload["USE_CHAT_TEMPLATE"], "true" if spec else "false"), + "AgentX client": (benchmark.get("command", "").endswith("srt_agentic.sh"), agentic), } - if "CONC" in workload: - expected["CONC"] = (str(workload["CONC"]), environment["CONC"]) + if not agentic: + expected["USE_CHAT_TEMPLATE"] = (workload["USE_CHAT_TEMPLATE"], "true" if spec else "false") + for name in ("ISL", "OSL", "RANDOM_RANGE_RATIO"): + expected[name] = (str(workload[name]), environment[name]) + # A variant that names its point, or the host budget it sizes, must match the matrix. + for name in ("CONC", "KV_OFFLOADING", "TOTAL_CPU_DRAM_GB"): + if name in workload: + expected[name] = (str(workload[name]), environment[name]) if engine == "atom": # Native ATOM derives -tp from the aggregate worker's GPU allocation. expected["ATOM TP"] = (role["gpus"], int(environment["TP"])) - for name in ("ISL", "OSL", "RANDOM_RANGE_RATIO"): - expected[name] = (str(workload[name]), environment[name]) - # Multi-node and AgentX workloads use their existing connector. - for name, value in { - "PP_SIZE": "1", - "DCP_SIZE": "1", - "PCP_SIZE": "1", - "IS_AGENTIC": "0", - }.items(): + for name, value in {"PP_SIZE": "1", "DCP_SIZE": "1", "PCP_SIZE": "1"}.items(): expected[name] = (environment[name], value) for name, (actual, wanted) in expected.items(): if actual != wanted: @@ -141,14 +140,18 @@ def runtime_arguments(config: str, environment: Mapping[str, str]) -> list[str]: # flags. Runtime option mappings therefore need individual leaf sets. for key, value in options.items(): overrides += ["--set", f"srun_options.{key}={json.dumps(value)}"] - for name in ( + agentic = environment["IS_AGENTIC"] == "1" + names = [ "CONC", "RESULT_FILENAME", "GPU_MONITOR_INTERVAL", "RUN_EVAL", "EVAL_ONLY", "FRAMEWORK", - ): + ] + if agentic: + names += ["MODEL_PREFIX", "PRECISION", "DURATION", "TP", "PP_SIZE", "PCP_SIZE"] + for name in names: value = environment[name] if not value: raise ValueError(f"Missing runtime input: {name}") @@ -157,6 +160,10 @@ def runtime_arguments(config: str, environment: Mapping[str, str]) -> list[str]: if name == "CONC" and name in recipe["benchmark"]["env"]: continue overrides += ["--set", f"benchmark.env.{name}={json.dumps(value)}"] + if agentic: + # The aggregated result lands where fixed-sequence results do. + overrides += ["--set", 'benchmark.env.AGENTIC_OUTPUT_DIR="/logs"'] + return [*overrides, "--set", 'benchmark.env.RESULT_DIR="/logs/agentic"'] if environment["EVAL_ONLY"] == "true": context = int(environment["MAX_MODEL_LEN"]) if context <= 0: diff --git a/runners/launch_b200-cw.sh b/runners/launch_b200-cw.sh index 655599e755..60d86d26ff 100644 --- a/runners/launch_b200-cw.sh +++ b/runners/launch_b200-cw.sh @@ -6,7 +6,7 @@ check_env_vars IS_MULTINODE IS_AGENTIC EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi diff --git a/runners/launch_b200-nb.sh b/runners/launch_b200-nb.sh index 419af005de..cece9b5801 100644 --- a/runners/launch_b200-nb.sh +++ b/runners/launch_b200-nb.sh @@ -6,7 +6,7 @@ check_env_vars IS_MULTINODE IS_AGENTIC EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index a041b9f403..8560740d7d 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -50,7 +50,7 @@ if uses_native_srt_lane; then LAUNCH_PATH="native-srt" elif [[ "$IS_MULTINODE" == "true" ]]; then LAUNCH_PATH="multinode-srt" -elif [[ "$IS_AGENTIC" == "0" ]]; then +elif [[ "$IS_AGENTIC" == "0" || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE LAUNCH_PATH="native-single-node" else @@ -155,6 +155,8 @@ fi if [[ "$LAUNCH_PATH" == native-single-node ]]; then HF_HUB_CACHE_MOUNT=/data/home/sa-shared/gharunners/hf-hub-cache SRT_MODEL_PATH="$MODEL_PATH" + # Models not staged locally resolve through the Hugging Face cache mount. + [[ "$SRT_MODEL_PATH" == /* ]] || SRT_MODEL_PATH="hf:$MODEL" SRT_SQUASH_FILE="$B200_SQUASH_DIR/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" launch_srt_single_node b200-nscale-slurm \ --var SLURM_ACCOUNT "$SLURM_ACCOUNT" --var SLURM_PARTITION "$SLURM_PARTITION" diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index c5f33162c4..ba0269b9ca 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -127,7 +127,7 @@ if [[ "$IS_MULTINODE" == true ]]; then elif [[ -n "${BENCH_SCRIPT_OVERRIDE:-}" ]]; then # SPEED-Bench collectors explicitly supply their script outside this migration. EXECUTION_PATH=script -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi diff --git a/runners/launch_h100-cw.sh b/runners/launch_h100-cw.sh index 693211a505..a5cb2ae7f3 100644 --- a/runners/launch_h100-cw.sh +++ b/runners/launch_h100-cw.sh @@ -6,7 +6,7 @@ check_env_vars IS_MULTINODE IS_AGENTIC EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 18a153cccf..31058b0cc2 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -17,7 +17,7 @@ set -x EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi diff --git a/runners/launch_h200-cw.sh b/runners/launch_h200-cw.sh index d101457309..4ebe6b68bc 100644 --- a/runners/launch_h200-cw.sh +++ b/runners/launch_h200-cw.sh @@ -6,7 +6,7 @@ check_env_vars IS_MULTINODE IS_AGENTIC EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 55651913ba..5965c9176c 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -18,12 +18,13 @@ source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" || exit 1 EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi if [[ "$EXECUTION_PATH" == native-single-node ]]; then + SRT_MODEL_PATH="hf:$MODEL" SRT_SQUASH_FILE="/data/containers/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" launch_srt_single_node h200-dgxc-slurm \ --var SLURM_ACCOUNT "$SLURM_ACCOUNT" --var SLURM_PARTITION "$SLURM_PARTITION" \ diff --git a/runners/launch_mi300x-amd.sh b/runners/launch_mi300x-amd.sh index e881f41dc2..f413318c5b 100644 --- a/runners/launch_mi300x-amd.sh +++ b/runners/launch_mi300x-amd.sh @@ -8,7 +8,7 @@ set -eo pipefail EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi diff --git a/runners/launch_mi325x-amds.sh b/runners/launch_mi325x-amds.sh index 8063915ea5..1b6f29c993 100644 --- a/runners/launch_mi325x-amds.sh +++ b/runners/launch_mi325x-amds.sh @@ -8,7 +8,7 @@ set -eo pipefail EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index c97c7beb55..b3eec28469 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -7,7 +7,7 @@ check_env_vars EVAL_ONLY IS_AGENTIC IS_MULTINODE KEEP_LOGS RUN_EVAL EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 68aedf6a25..4c00625543 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -150,7 +150,7 @@ apply_srt_recipe() { "$config" "$framework" -- "$@" } -# One native submission per fixed-sequence matrix point, shared across Slurm pools. +# One native submission per fixed-sequence or AgentX matrix point, shared across Slurm pools. launch_srt_single_node() { set -eo pipefail local profile="$1" @@ -216,6 +216,10 @@ launch_srt_single_node() { [[ -f "$artifact" ]] || continue copy_to_workspace "$artifact" "$GITHUB_WORKSPACE/$(basename "$artifact")" || rc=1 done + # AgentX uploads its raw replay artifacts and power window from results/. + if [[ -d "$SRT_JOB_OUTPUT/logs/agentic" ]]; then + cp -r "$SRT_JOB_OUTPUT/logs/agentic" "$GITHUB_WORKSPACE/results" || rc=1 + fi fi exit "$rc" } diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index c52883d20e..84ae7d0567 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -20,7 +20,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] BENCHMARK_LIB = REPO_ROOT / "benchmarks" / "benchmark_lib.sh" -MULTINODE_AGENTIC_SCRIPT = REPO_ROOT / "benchmarks/multi_node/agentic_srt.sh" +MULTINODE_AGENTIC_SCRIPT = REPO_ROOT / "benchmarks/srt_agentic.sh" @pytest.fixture(autouse=True) diff --git a/utils/test_srt_single_node.py b/utils/test_srt_single_node.py index 979f286daf..3717668320 100644 --- a/utils/test_srt_single_node.py +++ b/utils/test_srt_single_node.py @@ -335,7 +335,7 @@ def test_pool_launcher_stages_artifacts_and_propagates_failure(point, tmp_path, env.pop("SRT_RECIPE") if failure == "agentic": env.update(IS_AGENTIC="1", SCENARIO_SUBDIR="agentic/", EXP_NAME="fixture_agentic", - RUNNER_NAME="fixture_00", SRT_RECIPE="unused.yaml") + RUNNER_NAME="fixture_00", SRT_RECIPE="") # not yet ported: legacy script result = subprocess.run( ["bash", str(ROOT / f"runners/launch_{pool}.sh")], cwd=tmp_path, env=env, capture_output=True, text=True, timeout=30, From 765e70fdc60b2523c340fd3c749443fe8ad01042 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:01:11 -0500 Subject: [PATCH 002/105] fix(amd): drop a duplicate DSV4 prefill key and restore the MI355X Qwen3.5 fixed-sequence recipe srt-slurm parses recipes strictly and rejected the repeated disable-cuda-graph key. Removing the unported MI355X Qwen3.5 AgentX recipe had also deleted the fixed-sequence recipe beside it. --- .../mi355x-fp4/agentx/disagg-umbp-dspark.yaml | 1 - .../qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml | 75 +++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml index 4429f029e3..d0402412f9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml @@ -163,7 +163,6 @@ base: context-length: 1048576 enable-metrics: true disable-cuda-graph: true - disable-cuda-graph: true enable-unified-cache-external-linker: true unified-cache-external-linker-backend: mori hicache-storage-backend-extra-config: '{"standalone_startup_timeout_ms":120000}' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml new file mode 100644 index 0000000000..244b8d64dc --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml @@ -0,0 +1,75 @@ +base: + schema: 2 + name: qwen3.5-fp4-mi355x-sglang-mtp-8k1k + model: + path: hf:amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 + container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + args: + trust-remote-code: true + tensor-parallel-size: 2 + attention-backend: aiter + mem-fraction-static: 0.8 + model-loader-extra-config: '{"enable_multithread_load": true}' + watchdog-timeout: 1200 + disable-radix-cache: true + max-running-requests: 4 + page-size: 16 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + kv-cache-dtype: fp8_e4m3 + data-parallel-size: 1 + expert-parallel-size: 1 + served-model-name: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 + enable-metrics: false + env: + SGLANG_USE_AITER: '1' + SGLANG_USE_AITER_UNIFIED_ATTN: '1' + AITER_FLYDSL_FORCE: '1' + SGLANG_MAMBA_SSM_DTYPE: bfloat16 + ROCM_QUICK_REDUCE_QUANTIZATION: INT8 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/single_node/srt_fixed_sequence.sh + env: + MODEL: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 + ISL: '8192' + OSL: '1024' + RANDOM_RANGE_RATIO: '0.8' + USE_CHAT_TEMPLATE: 'true' +zip_override_tp2_ep1: + roles: + agg: + args: + max-running-requests: [4, 8, 16, 32, 64, 128] + benchmark: + env: + CONC: ['4', '8', '16', '32', '64', '128'] +zip_override_tp4_ep1: + roles: + agg: + gpus: 4 + args: + max-running-requests: [4, 8, 16] + tensor-parallel-size: 4 + benchmark: + env: + CONC: ['4', '8', '16'] From f0d421961ceb95e0cbde0a8a285f4340fc24790d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:09:34 -0500 Subject: [PATCH 003/105] feat(srt): accept vLLM points in the single-node adapter vLLM points validate their topology as tensor x data parallel GPUs, with DP attention as data-parallel ranks and expert parallelism as enable-expert-parallel, and eval-only runs set max-model-len. --- infx/srt_slurm/single_node.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/infx/srt_slurm/single_node.py b/infx/srt_slurm/single_node.py index a4a5a14fef..8a2c1dcae3 100644 --- a/infx/srt_slurm/single_node.py +++ b/infx/srt_slurm/single_node.py @@ -37,6 +37,14 @@ def parallelism_constraints( "pipeline_parallel_size": (args.get("pipeline_parallel_size", 1), 1), "DP_ATTENTION": (args.get("enable_attention_dp", False), dp_attention), } + if engine == "vllm": + # vLLM spreads DP attention across data-parallel ranks of tensor size 1. + data_parallel = args.get("data-parallel-size", 1) + return { + "tensor x data parallel": (args.get("tensor-parallel-size", 1) * data_parallel, tp), + "DP_ATTENTION": (data_parallel > 1, dp_attention), + "enable-expert-parallel": (args.get("enable-expert-parallel", False), ep > 1), + } if engine == "atom": if ep not in {1, tp}: raise ValueError("ATOM expert parallelism must be 1 or TP") @@ -77,7 +85,7 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N workload = benchmark["env"] engine_config = recipe["engine"] engine = engine_config["type"] if isinstance(engine_config, dict) else engine_config - if environment["FRAMEWORK"] not in {"sglang", "trt", "atom"}: + if environment["FRAMEWORK"] not in {"sglang", "trt", "atom", "vllm"}: raise ValueError(f"Unsupported single-node framework: {environment['FRAMEWORK']!r}") spec = spec_parameters(role, engine) if spec and spec["method"] not in {"eagle", "nextn", "mtp", "dspark"}: @@ -172,6 +180,7 @@ def runtime_arguments(config: str, environment: Mapping[str, str]) -> list[str]: "sglang": ("context-length",), "trt": ("max_seq_len", "max_num_tokens"), "atom": ("max-model-len",), + "vllm": ("max-model-len",), }[environment["FRAMEWORK"]] for key in context_keys: overrides += ["--set", f"roles.agg.args.{key}={context}"] From e9667ed42dfc52f95817c6d55c11980ea444e6a7 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:13:51 -0500 Subject: [PATCH 004/105] feat(agentx): port the DSV4.1 Flash B300 SGLang AgentX config to srt-slurm --- .../sglang/b300-fp4-mtp/agentic.yaml | 82 +++++++++++++++++++ configs/nvidia-master.yaml | 4 +- 2 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..7415460727 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,82 @@ +# DeepSeek-V4.1-Flash AgentX on B300 with SGLang native DSpark. The KV cache +# is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-b300-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:35ea4d321b0735051dcce2599fc495853ad1ef30f6a1908f0a75e74204362d14 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + # Overlap checkpoint reads with weight loading from local NVMe. + weight-loader-prefetch-checkpoints: true + chunked-prefill-size: 4096 + # Keep active decode requests progressing while long prefixes are queued. + prefill-decode-interval: 16 + # DSpark is the checkpoint's bundled draft; the block size is its only knob. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + # Admission never exceeds the captured decode graph batch. + cuda-graph-max-bs-decode: 64 + reasoning-parser: auto + tool-call-parser: auto + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV41_REASONING_EFFORT: high + # Row-sharded host Engram tables free HBM for the 1M-context working set. + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC, capped at the graph batch. SWA +# prefix tails are 64x CONC within [128, 4096]. TP2 raises the static fraction +# (0.80 from c16, 0.85 from c32) and interleaves decode more often from c32. +# Saturation points get a longer warmup drain. +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 128, max-running-requests: 2}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 128, max-running-requests: 4}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 256, max-running-requests: 8}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 512, max-running-requests: 16}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 1024, max-running-requests: 32}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 2048, max-running-requests: 64}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 4096, max-running-requests: 64}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 4096, max-running-requests: 64}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.7, swa-prefix-tails: 128, max-running-requests: 2}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.7, swa-prefix-tails: 128, max-running-requests: 4}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.7, swa-prefix-tails: 256, max-running-requests: 8}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.7, swa-prefix-tails: 512, max-running-requests: 16}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.8, swa-prefix-tails: 1024, max-running-requests: 32}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.85, swa-prefix-tails: 2048, max-running-requests: 64, prefill-decode-interval: 4}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.85, swa-prefix-tails: 4096, max-running-requests: 64, prefill-decode-interval: 4}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.85, swa-prefix-tails: 4096, max-running-requests: 64, prefill-decode-interval: 4}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index d3908eca37..46179b44a9 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8997,8 +8997,8 @@ dsv41flash-fp4-b300-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml } # Official SGLang nightly; Engram weights in host DRAM, native DSpark draft. dsv41flash-fp4-gb300-sglang-agentic-dspark: From bbe361cc5e2d1f8a163951fc52ee3d8462253670 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:19:29 -0500 Subject: [PATCH 005/105] feat(agentx): let srt-slurm AgentX recipes apply the chat template client-side Several legacy AMD AgentX scripts appended --apply-chat-template to the replay command. AIPERF_APPLY_CHAT_TEMPLATE=true in a recipe's benchmark env keeps that behavior on the shared srt-slurm client. --- benchmarks/srt_agentic.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchmarks/srt_agentic.sh b/benchmarks/srt_agentic.sh index c0f61583cb..1352ba010c 100644 --- a/benchmarks/srt_agentic.sh +++ b/benchmarks/srt_agentic.sh @@ -168,6 +168,10 @@ for index in "${!CONCURRENCIES[@]}"; do echo "Running agentic concurrency $concurrency of: ${CONCURRENCIES[*]}" build_replay_cmd "$RESULT_DIR" + # Recipes whose legacy launch rendered prompts client-side opt in here. + if [[ "${AIPERF_APPLY_CHAT_TEMPLATE:-}" == true ]]; then + REPLAY_CMD+=" --apply-chat-template" + fi run_agentic_replay_and_write_outputs "$RESULT_DIR" if [ "$index" -lt "$(( ${#CONCURRENCIES[@]} - 1 ))" ]; then From 4fce20b99c4e84b179aea9ad5043c5716b36c293 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:19:50 -0500 Subject: [PATCH 006/105] feat(agentx): run GB200/GB300 single-node AgentX natively and port DSV4.1 Flash SGLang there GB launchers submit recipe points through launch_srt_single_node with an aarch64 srt-slurm setup; the squash is imported on a compute tray first. --- .../sglang/gb200-fp4-mtp/agentic.yaml | 76 ++++++++++++++++++ .../sglang/gb300-fp4-mtp/agentic.yaml | 78 +++++++++++++++++++ configs/nvidia-master.yaml | 8 +- runners/launch_gb200-nv.sh | 11 +++ runners/launch_gb300-nv.sh | 12 +++ runners/slurm_utils.sh | 2 +- 6 files changed, 182 insertions(+), 5 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..95d6ef6eca --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml @@ -0,0 +1,76 @@ +# DeepSeek-V4.1-Flash AgentX on GB200 with SGLang native DSpark. The KV cache +# is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-gb200-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17@sha256:5921361fcf358cdde4df1968c941c14157f418613b099ad7f3e5aeed6427ae15 + precision: fp4 + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + # DSpark is the checkpoint's bundled draft; the block size is its only knob. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + reasoning-parser: auto + tool-call-parser: auto + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV41_REASONING_EFFORT: high + # Row-sharded host Engram tables free HBM for the 1M-context working set. + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. TP4: static fraction 0.70, 4096-token prefill chunks, +# 64-request graph batch. TP2 needs 0.92, 2048-token chunks, a 16-request graph +# batch and expandable segments to fit. Admission is 2x CONC, capped at the graph +# batch. SWA prefix tails are 64x CONC (TP2 128x), capped at 1024. Decode +# interleaving applies to TP2 and c16. Saturation points get a longer warmup drain. +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 64, cuda-graph-max-bs-decode: 64, max-running-requests: 2}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 128, cuda-graph-max-bs-decode: 64, max-running-requests: 4}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 256, cuda-graph-max-bs-decode: 64, max-running-requests: 8}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 512, cuda-graph-max-bs-decode: 64, max-running-requests: 16}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 64, max-running-requests: 32, prefill-decode-interval: 16}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 64, max-running-requests: 64}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 64, max-running-requests: 64}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 64, max-running-requests: 64}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 128, cuda-graph-max-bs-decode: 16, max-running-requests: 2, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 256, cuda-graph-max-bs-decode: 16, max-running-requests: 4, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 512, cuda-graph-max-bs-decode: 16, max-running-requests: 8, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 16, max-running-requests: 16, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 16, max-running-requests: 16, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 16, max-running-requests: 16, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 16, max-running-requests: 16, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 16, max-running-requests: 16, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..9b6b882706 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml @@ -0,0 +1,78 @@ +# DeepSeek-V4.1-Flash AgentX on GB300 with SGLang native DSpark. The KV cache +# is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-gb300-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40 + precision: fp4 + resources: + gpu_type: gb300 + gpus_per_node: 4 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + # Keep active decode requests progressing while long prefixes are queued. + prefill-decode-interval: 16 + mem-fraction-static: 0.8 + # DSpark is the checkpoint's bundled draft; the block size is its only knob. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + # Admission never exceeds the captured decode graph batch. + cuda-graph-max-bs-decode: 64 + reasoning-parser: auto + tool-call-parser: auto + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV41_REASONING_EFFORT: high + SGLANG_DSPARK_OPT_MARKOV_W2_BF16: 'True' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC, capped at the graph batch. SWA +# prefix tails are 64x CONC up to 4096 from c2. TP2 keeps row-sharded Engram +# tables in host DRAM (TP4 fits them in HBM) and doubles the prefill chunk from +# c64. Saturation points get a longer warmup drain. +override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 4096, max-running-requests: 2}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 4096, swa-prefix-tails: 128, max-running-requests: 4}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 4096, swa-prefix-tails: 256, max-running-requests: 8}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 4096, swa-prefix-tails: 512, max-running-requests: 16}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 4096, swa-prefix-tails: 1024, max-running-requests: 32}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 4096, swa-prefix-tails: 2048, max-running-requests: 64}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 8192, swa-prefix-tails: 4096, max-running-requests: 64}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 8192, swa-prefix-tails: 4096, max-running-requests: 64}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, max-running-requests: 2}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, swa-prefix-tails: 128, max-running-requests: 4}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, swa-prefix-tails: 256, max-running-requests: 8}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, swa-prefix-tails: 512, max-running-requests: 16}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, swa-prefix-tails: 1024, max-running-requests: 32}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, swa-prefix-tails: 2048, max-running-requests: 64}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, swa-prefix-tails: 4096, max-running-requests: 64}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, swa-prefix-tails: 4096, max-running-requests: 64}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 46179b44a9..99ee1b3432 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8411,8 +8411,8 @@ dsv41flash-fp4-gb200-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - - { tp: 2, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml } dsv41flash-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:deepseekv41-flash-0909 @@ -9013,5 +9013,5 @@ dsv41flash-fp4-gb300-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index f26da84aff..66a0996ba3 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -95,6 +95,17 @@ import_squash() { ) || exit 1 } +# Single-tray points with an srt-slurm recipe run natively on the aarch64 trays. +if [[ "$IS_MULTINODE" != true && -n "${SRT_RECIPE:-}" ]]; then + HF_HUB_CACHE_MOUNT="/mnt/lustre01/users-public/sa-shared/hf-hub-cache" + SRT_MODEL_PATH="hf:$MODEL" + SRT_SQUASH_FILE="$SQUASH_DIR/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + import_squash "$SRT_SQUASH_FILE" "$IMAGE" + SRT_SETUP_ARCH=aarch64 launch_srt_single_node gb200-nv \ + --var SLURM_ACCOUNT "$SLURM_ACCOUNT" --var SLURM_PARTITION "$SLURM_PARTITION" + exit $? +fi + # Direct single-tray AgentX uses the existing shared image and HF caches. if [[ "$MODEL_PREFIX" == "dsv41flash" && ( "$FRAMEWORK" == "vllm" || "$FRAMEWORK" == "sglang" ) && "${IS_MULTINODE}" != "true" ]]; then check_env_vars SPEC_DECODING diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 2e5d4069dc..7d579742b0 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -105,6 +105,18 @@ import_squash() { } import_squash "$SQUASH_FILE" "$IMAGE" +# Single-tray points with an srt-slurm recipe run natively on the aarch64 trays. +if [[ "$IS_MULTINODE" != true && -n "${SRT_RECIPE:-}" ]]; then + HF_HUB_CACHE_MOUNT="$HF_HUB_CACHE_HOST_PATH" + SRT_MODEL_PATH="hf:$MODEL" + SRT_SQUASH_FILE="$SQUASH_FILE" + SRT_SETUP_ARCH=aarch64 launch_srt_single_node gb300-nv \ + --var SLURM_ACCOUNT "$SLURM_ACCOUNT" --var SLURM_PARTITION "$SLURM_PARTITION" \ + --var AIPERF_MMAP_CACHE_HOST_PATH "$AIPERF_MMAP_CACHE_HOST_PATH" \ + --var HF_HUB_CACHE_HOST_PATH "$HF_HUB_CACHE_HOST_PATH" \ + --var DYNAMO_WHEELS_CACHE_HOST_PATH "$DYNAMO_WHEELS_CACHE_HOST_PATH" + exit $? +fi # Keep this branch before the nginx import and srtctl setup. if [[ "$MODEL_PREFIX" == "dsv41flash" && ( "$FRAMEWORK" == "vllm" || "$FRAMEWORK" == "sglang" ) && "${IS_MULTINODE}" != "true" ]]; then check_env_vars SPEC_DECODING diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 4c00625543..50370edfeb 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -193,7 +193,7 @@ launch_srt_single_node() { --var SRT_DEFAULT_TIME_LIMIT "$SALLOC_TIME_LIMIT" \ --model "hf:$MODEL" "$SRT_MODEL_PATH" --container "$IMAGE" "$SRT_CONTAINER" \ --mount "$HF_HUB_CACHE_MOUNT" "$HF_HUB_CACHE" --exclusive "$@" - run_srt_setup ARCH=x86_64 + run_srt_setup "ARCH=${SRT_SETUP_ARCH:-x86_64}" SRT_JOB_ID="" SRT_JOB_OUTPUT="" From 90dc9ef714cef1a2074cc97f5759ca7dbb349697 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:19:29 -0500 Subject: [PATCH 007/105] feat(agentx): let srt-slurm AgentX recipes apply the chat template client-side Several legacy AMD AgentX scripts appended --apply-chat-template to the replay command. AIPERF_APPLY_CHAT_TEMPLATE=true in a recipe's benchmark env keeps that behavior on the shared srt-slurm client. --- benchmarks/srt_agentic.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchmarks/srt_agentic.sh b/benchmarks/srt_agentic.sh index c0f61583cb..1352ba010c 100644 --- a/benchmarks/srt_agentic.sh +++ b/benchmarks/srt_agentic.sh @@ -168,6 +168,10 @@ for index in "${!CONCURRENCIES[@]}"; do echo "Running agentic concurrency $concurrency of: ${CONCURRENCIES[*]}" build_replay_cmd "$RESULT_DIR" + # Recipes whose legacy launch rendered prompts client-side opt in here. + if [[ "${AIPERF_APPLY_CHAT_TEMPLATE:-}" == true ]]; then + REPLAY_CMD+=" --apply-chat-template" + fi run_agentic_replay_and_write_outputs "$RESULT_DIR" if [ "$index" -lt "$(( ${#CONCURRENCIES[@]} - 1 ))" ]; then From 5374133476f6c23009f1d53af70dd8e554618bb9 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:19:29 -0500 Subject: [PATCH 008/105] feat(agentx): let srt-slurm AgentX recipes apply the chat template client-side Several legacy AMD AgentX scripts appended --apply-chat-template to the replay command. AIPERF_APPLY_CHAT_TEMPLATE=true in a recipe's benchmark env keeps that behavior on the shared srt-slurm client. --- benchmarks/srt_agentic.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchmarks/srt_agentic.sh b/benchmarks/srt_agentic.sh index c0f61583cb..1352ba010c 100644 --- a/benchmarks/srt_agentic.sh +++ b/benchmarks/srt_agentic.sh @@ -168,6 +168,10 @@ for index in "${!CONCURRENCIES[@]}"; do echo "Running agentic concurrency $concurrency of: ${CONCURRENCIES[*]}" build_replay_cmd "$RESULT_DIR" + # Recipes whose legacy launch rendered prompts client-side opt in here. + if [[ "${AIPERF_APPLY_CHAT_TEMPLATE:-}" == true ]]; then + REPLAY_CMD+=" --apply-chat-template" + fi run_agentic_replay_and_write_outputs "$RESULT_DIR" if [ "$index" -lt "$(( ${#CONCURRENCIES[@]} - 1 ))" ]; then From 76e3cc509d2d4b86cbc92f4137ecc05d59569cfe Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:21:09 -0500 Subject: [PATCH 009/105] feat(agentx): let srt-slurm AgentX recipes set the AIPerf benchmark grace period The GLM-5.2 MI325X legacy script bounded the post-window drain with --benchmark-grace-period 1800. AIPERF_BENCHMARK_GRACE_PERIOD in a recipe's benchmark env passes it through the shared client. --- benchmarks/srt_agentic.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchmarks/srt_agentic.sh b/benchmarks/srt_agentic.sh index 1352ba010c..09aedbc1b9 100644 --- a/benchmarks/srt_agentic.sh +++ b/benchmarks/srt_agentic.sh @@ -172,6 +172,10 @@ for index in "${!CONCURRENCIES[@]}"; do if [[ "${AIPERF_APPLY_CHAT_TEMPLATE:-}" == true ]]; then REPLAY_CMD+=" --apply-chat-template" fi + # Bounded post-window drain for long responses admitted near the end. + if [[ -n "${AIPERF_BENCHMARK_GRACE_PERIOD:-}" ]]; then + REPLAY_CMD+=" --benchmark-grace-period $AIPERF_BENCHMARK_GRACE_PERIOD" + fi run_agentic_replay_and_write_outputs "$RESULT_DIR" if [ "$index" -lt "$(( ${#CONCURRENCIES[@]} - 1 ))" ]; then From 307487b46832031a8bdf50d530d1b54cbe55a3b5 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:21:46 -0500 Subject: [PATCH 010/105] feat(agentx): port the DSV4.1 Flash B200 vLLM AgentX config to srt-slurm --- .../dsv41flash/vllm/b200-fp4-mtp/agentic.yaml | 76 +++++++++++++++++++ configs/nvidia-master.yaml | 4 +- 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..9ed63491af --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,76 @@ +# DeepSeek-V4.1-Flash AgentX on B200 with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). Engram weights use +# UVA DRAM; the KV cache is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-b200-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + engram-config: '{"cpu_offload":true}' + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + disable-uvicorn-access-log: true + env: + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Graph capture starts at 64 tokens and doubles until it +# covers CONC x (1 + 5 drafts), up to 2048. TP2 leaves ~145 GiB of weights per +# rank, so it caps batched tokens at 4096 (the indexer's 1M-wide logits buffer), +# bounds the scheduler at 2x CONC within [16, 256] (FlashInfer autotune fails +# below 16) and stops capturing above 512 tokens. +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '1'}}} +override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '2'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '4'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '8'}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 128}}}, benchmark: {env: {CONC: '16'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 256}}}, benchmark: {env: {CONC: '32'}}} +override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '64'}}} +override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 1024}}}, benchmark: {env: {CONC: '128'}}} +override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '1'}}} +override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '2'}}} +override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '4'}}} +override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} +override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 128, max-num-batched-tokens: 4096, max-num-seqs: 32}}}, benchmark: {env: {CONC: '16'}}} +override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 256, max-num-batched-tokens: 4096, max-num-seqs: 64}}}, benchmark: {env: {CONC: '32'}}} +override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 128}}}, benchmark: {env: {CONC: '64'}}} +override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 256}}}, benchmark: {env: {CONC: '128'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 46179b44a9..a2c7bd1bd1 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8445,8 +8445,8 @@ dsv41flash-fp4-b200-vllm-agentic-dspark: - dram-utilization: 0.80 search-space: # Engram weights use UVA DRAM; the KV cache stays GPU-resident. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml } + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml } # SGLang arm for DeepSeek-V4.1-Flash AgentX on B200, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). From a743687d5d56c4d8c3d2c641be94cc8258f2084c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:21:55 -0500 Subject: [PATCH 011/105] feat(srt): force ATOM AgentX golden acceptance with its server flag ATOM pins acceptance with --spec-decode-acceptance-length rather than an environment variable. The adapter now sets it from the golden curve for AgentX throughput and removes it for evals, reading the draft model for the MiniMax GQA curve and the probabilistic sampler for Kimi DSpark. --- infx/srt_slurm/synthetic_acceptance.py | 15 ++++++++++++++- utils/test_synthetic_acceptance.py | 22 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/infx/srt_slurm/synthetic_acceptance.py b/infx/srt_slurm/synthetic_acceptance.py index be7232eb12..724eb535d6 100644 --- a/infx/srt_slurm/synthetic_acceptance.py +++ b/infx/srt_slurm/synthetic_acceptance.py @@ -27,6 +27,7 @@ "dynamo-sglang": "sglang", "trt": "trtllm", "dynamo-trt": "trtllm", + "atom": "atom", } SGLANG_VARIABLES = ( "SGLANG_SIMULATE_ACC_LEN", @@ -42,10 +43,15 @@ def spec_parameters(role: Mapping[str, Any], engine: str) -> dict[str, Any]: method = args.get("method") if not method: return {} - return { + spec = { "method": str(method).lower(), "num_speculative_tokens": args.get("num-speculative-tokens"), + "model": args.get("draft-model", ""), } + if spec["method"] == "dspark": + # ATOM DSpark verifies with probabilistic block rejection sampling. + spec["draft_sample_method"] = "probabilistic" + return spec if engine == "vllm": raw = args.get("speculative-config") if raw is None: @@ -181,6 +187,13 @@ def build_overrides( "--set", f"{prefix}.args.speculative-config={json.dumps(worker_spec)}", ] + elif engine == "atom": + # ATOM forces acceptance with a server flag rather than environment. + key = "spec-decode-acceptance-length" + if al is not None and worker_spec: + overrides += ["--set", f"{prefix}.args.{key}={al:g}"] + elif key in (role.get("args") or {}): + overrides += ["--unset", f"{prefix}.args.{key}"] elif al is not None and worker_spec: values = ( (f"{al:g}", "match-expected", "real-draft-token") diff --git a/utils/test_synthetic_acceptance.py b/utils/test_synthetic_acceptance.py index ffa8b01aa0..a3ae8f92f2 100644 --- a/utils/test_synthetic_acceptance.py +++ b/utils/test_synthetic_acceptance.py @@ -267,6 +267,28 @@ def test_real_runs_clear_synthetic_without_a_curve( assert result["environment"]["KEEP"] == "yes" +@pytest.mark.parametrize( + ("prefix", "args", "expected"), + [ + ("dsv4", {"method": "dspark"}, 2.7), + ("kimik3", {"method": "dspark", "draft-model": "Inferact/Kimi-K3-DSpark"}, 2.9), + ("minimaxm3", {"method": "eagle3", "draft-model": "Inferact/MiniMax-M3-EAGLE3-GQA"}, 2.6), + ], +) +def test_atom_forces_golden_acceptance_by_server_flag( + golden_dir: Path, prefix: str, args: dict[str, Any], expected: float +) -> None: + recipe = {"roles": {"agg": {"args": {**args, "num-speculative-tokens": 3}, "env": {}}}} + env = {**ENV, "MODEL_PREFIX": prefix} + result = apply_native(recipe, build_overrides(recipe, "atom", env, golden_dir=golden_dir)) + assert result["roles"]["agg"]["args"]["spec-decode-acceptance-length"] == expected + # Evals verify real drafts, so a recipe-pinned acceptance length is removed. + evaluated = apply_native( + result, build_overrides(result, "atom", {**env, "EVAL_ONLY": "true"}, golden_dir=golden_dir) + ) + assert "spec-decode-acceptance-length" not in evaluated["roles"]["agg"]["args"] + + @pytest.mark.parametrize( "curve", [ From 0b6114dda853948876f1ba65e1160ca0873e7567 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:21:55 -0500 Subject: [PATCH 012/105] feat(srt): accept draft_model speculation in the single-node adapter DeepSeek-V4 configs mark their bundled DSpark draft as draft_model; the recipe still speculates natively, so the point binds as speculative. --- infx/srt_slurm/single_node.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/infx/srt_slurm/single_node.py b/infx/srt_slurm/single_node.py index 8a2c1dcae3..052a427e7a 100644 --- a/infx/srt_slurm/single_node.py +++ b/infx/srt_slurm/single_node.py @@ -104,7 +104,11 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N "roles": (set(recipe["roles"]), {"agg"}), "benchmark type": (benchmark["type"], "custom"), "benchmark MODEL": (workload["MODEL"], environment["MODEL"]), - "SPEC_DECODING": (speculation, environment["SPEC_DECODING"]), + # draft_model names a bundled or separate draft; its recipes speculate natively. + "SPEC_DECODING": ( + speculation, + "mtp" if environment["SPEC_DECODING"] == "draft_model" else environment["SPEC_DECODING"], + ), "AgentX client": (benchmark.get("command", "").endswith("srt_agentic.sh"), agentic), } if not agentic: From d0f755bed686b24e37dfdeb4c58d22d4daa17868 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:22:22 -0500 Subject: [PATCH 013/105] fix(mi355x): mount the shared HF cache for native AgentX checkpoints The legacy MI355X AgentX path read DeepSeek-V4-Pro (vLLM/ATOM), DeepSeek-V4.1 Flash, MiniMax-M3 and GLM-5.2-FP8 from /it-share rather than node-local NVMe. The native srt-slurm path keeps that mount for those AgentX checkpoints. --- runners/launch_mi355x-amds.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index b3eec28469..38b15fe1a9 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -15,6 +15,17 @@ if [[ "$EXECUTION_PATH" == native-single-node ]]; then check_env_vars GITHUB_WORKSPACE MODEL IMAGE source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" || exit 1 export HF_HUB_CACHE_MOUNT=/var/lib/hf-hub-cache/ + # AgentX checkpoints the legacy scripts read from the shared NFS cache. + if [[ "$IS_AGENTIC" == 1 ]]; then + case "$MODEL" in + MiniMaxAI/MiniMax-M3*|amd/MiniMax-M3*|zai-org/GLM-5.2-FP8|deepseek-ai/DeepSeek-V4.1-Flash) + export HF_HUB_CACHE_MOUNT=/it-share/hf-hub-cache/ ;; + deepseek-ai/DeepSeek-V4-Pro|deepseek-ai/DeepSeek-V4-Pro-0813) + if [[ "$FRAMEWORK" == vllm || "$FRAMEWORK" == atom ]]; then + export HF_HUB_CACHE_MOUNT=/it-share/hf-hub-cache/ + fi ;; + esac + fi export SRT_MODEL_PATH="hf:$MODEL" export SALLOC_TIME_LIMIT=500 export SRT_SRUN_OPTIONS='{"container-remap-root":"", "container-writable":""}' From e664992e4b3a2747fb649c5529aea4feebd0121e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:22:29 -0500 Subject: [PATCH 014/105] feat(agentx): port the Qwen3.5 MI300X SGLang AgentX config to srt-slurm --- .../sglang/mi300x-fp8-mtp/agentic.yaml | 71 +++++++++++++++++++ configs/amd-master.yaml | 4 +- 2 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..e9cc55d5fd --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml @@ -0,0 +1,71 @@ +# Qwen3.5 FP8 AgentX on MI300X with SGLang native EAGLE MTP. The KV cache is +# GPU-resident. +base: + schema: 2 + name: qwen3.5-fp8-mi300x-sglang-agentic + model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730 + precision: fp8 + resources: + gpu_type: mi300x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + tokenizer-path: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + data-parallel-size: 1 + attention-backend: aiter + enable-aiter-allreduce-fusion: true + mamba-ssm-dtype: bfloat16 + tokenizer-worker-num: 6 + max-prefill-tokens: 32768 + scheduler-recv-interval: 30 + mem-fraction-static: 0.75 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-metrics: true + enable-cache-report: true + env: + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + # 256k-capped corpus: the unfiltered corpus exceeds this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. AgentX concurrency counts live session trees, so +# admission is 2x CONC; the graph batch is CONC, capped at 64. +override_tp8_ep1_c4: {roles: {agg: {args: {expert-parallel-size: 1, cuda-graph-max-bs: 4, max-running-requests: 8}}}, benchmark: {env: {CONC: '4'}}} +override_tp8_ep1_c16: {roles: {agg: {args: {expert-parallel-size: 1, cuda-graph-max-bs: 16, max-running-requests: 32}}}, benchmark: {env: {CONC: '16'}}} +override_tp8_ep1_c20: {roles: {agg: {args: {expert-parallel-size: 1, cuda-graph-max-bs: 20, max-running-requests: 40}}}, benchmark: {env: {CONC: '20'}}} +override_tp8_ep1_c24: {roles: {agg: {args: {expert-parallel-size: 1, cuda-graph-max-bs: 24, max-running-requests: 48}}}, benchmark: {env: {CONC: '24'}}} +override_tp8_ep1_c32: {roles: {agg: {args: {expert-parallel-size: 1, cuda-graph-max-bs: 32, max-running-requests: 64}}}, benchmark: {env: {CONC: '32'}}} +override_tp8_ep8_c4: {roles: {agg: {args: {expert-parallel-size: 8, cuda-graph-max-bs: 4, max-running-requests: 8}}}, benchmark: {env: {CONC: '4'}}} +override_tp8_ep8_c16: {roles: {agg: {args: {expert-parallel-size: 8, cuda-graph-max-bs: 16, max-running-requests: 32}}}, benchmark: {env: {CONC: '16'}}} +override_tp8_ep8_c20: {roles: {agg: {args: {expert-parallel-size: 8, cuda-graph-max-bs: 20, max-running-requests: 40}}}, benchmark: {env: {CONC: '20'}}} +override_tp8_ep8_c24: {roles: {agg: {args: {expert-parallel-size: 8, cuda-graph-max-bs: 24, max-running-requests: 48}}}, benchmark: {env: {CONC: '24'}}} +override_tp8_ep8_c32: {roles: {agg: {args: {expert-parallel-size: 8, cuda-graph-max-bs: 32, max-running-requests: 64}}}, benchmark: {env: {CONC: '32'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 89cd673dac..201631f2bc 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -488,8 +488,8 @@ qwen3.5-fp8-mi300x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32] } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32] } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml } dsr1-fp8-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 From 69b585564d9466979660c820e9313fcb0fe95ff6 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:22:29 -0500 Subject: [PATCH 015/105] feat(agentx): port the Qwen3.5 MI325X SGLang AgentX config to srt-slurm --- .../sglang/mi325x-fp8-mtp/agentic.yaml | 102 ++++++++++++++++++ configs/amd-master.yaml | 10 +- 2 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..da992cb3e0 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml @@ -0,0 +1,102 @@ +# Qwen3.5 FP8 AgentX on MI325X with SGLang native EAGLE MTP. The KV cache is +# GPU-resident. +base: + schema: 2 + name: qwen3.5-fp8-mi325x-sglang-agentic + model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang:v0.5.16-rocm720-mi30x + precision: fp8 + resources: + gpu_type: mi325x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + tokenizer-path: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + data-parallel-size: 1 + attention-backend: aiter + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + model-loader-extra-config: '{"enable_multithread_load": true}' + watchdog-timeout: 1200 + enable-aiter-allreduce-fusion: true + page-size: 16 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + scheduler-recv-interval: 30 + stream-interval: 50 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-metrics: true + enable-cache-report: true + env: + PYTHONNOUSERSITE: '1' + SGLANG_USE_AITER: '1' + SGLANG_USE_AITER_UNIFIED_ATTN: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + # 256k-capped corpus: the unfiltered corpus exceeds this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. AgentX concurrency counts live session trees, so +# admission is 2x CONC; the graph batch is CONC, capped at 64. With EAGLE, +# SGLang reserves 15% of the static budget for the draft model: TP2 needs 0.95 +# (an effective 0.8075) to fit weights, hybrid state and KV; TP4/TP8 use 0.80. +# Tokenizer workers run only from TP4. +override_tp2_ep2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.95, cuda-graph-max-bs: 1, max-running-requests: 2}}}, benchmark: {env: {CONC: '1'}}} +override_tp2_ep2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.95, cuda-graph-max-bs: 4, max-running-requests: 8}}}, benchmark: {env: {CONC: '4'}}} +override_tp2_ep2_c6: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.95, cuda-graph-max-bs: 6, max-running-requests: 12}}}, benchmark: {env: {CONC: '6'}}} +override_tp4_ep1_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 1, max-running-requests: 2}}}, benchmark: {env: {CONC: '1'}}} +override_tp4_ep1_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 8, max-running-requests: 16}}}, benchmark: {env: {CONC: '8'}}} +override_tp4_ep1_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 16, max-running-requests: 32}}}, benchmark: {env: {CONC: '16'}}} +override_tp4_ep1_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 32, max-running-requests: 64}}}, benchmark: {env: {CONC: '32'}}} +override_tp4_ep1_c40: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 40, max-running-requests: 80}}}, benchmark: {env: {CONC: '40'}}} +override_tp4_ep1_c48: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 48, max-running-requests: 96}}}, benchmark: {env: {CONC: '48'}}} +override_tp4_ep4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 4, max-running-requests: 8}}}, benchmark: {env: {CONC: '4'}}} +override_tp4_ep4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 16, max-running-requests: 32}}}, benchmark: {env: {CONC: '16'}}} +override_tp4_ep4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 32, max-running-requests: 64}}}, benchmark: {env: {CONC: '32'}}} +override_tp4_ep4_c40: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 40, max-running-requests: 80}}}, benchmark: {env: {CONC: '40'}}} +override_tp4_ep4_c48: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 48, max-running-requests: 96}}}, benchmark: {env: {CONC: '48'}}} +override_tp8_ep1_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 1, max-running-requests: 2}}}, benchmark: {env: {CONC: '1'}}} +override_tp8_ep1_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 4, max-running-requests: 8}}}, benchmark: {env: {CONC: '4'}}} +override_tp8_ep1_c8: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 8, max-running-requests: 16}}}, benchmark: {env: {CONC: '8'}}} +override_tp8_ep1_c12: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 12, max-running-requests: 24}}}, benchmark: {env: {CONC: '12'}}} +override_tp8_ep1_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 16, max-running-requests: 32}}}, benchmark: {env: {CONC: '16'}}} +override_tp8_ep1_c24: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 24, max-running-requests: 48}}}, benchmark: {env: {CONC: '24'}}} +override_tp8_ep1_c32: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 32, max-running-requests: 64}}}, benchmark: {env: {CONC: '32'}}} +override_tp8_ep1_c40: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 40, max-running-requests: 80}}}, benchmark: {env: {CONC: '40'}}} +override_tp8_ep1_c48: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 48, max-running-requests: 96}}}, benchmark: {env: {CONC: '48'}}} +override_tp8_ep1_c64: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 64, max-running-requests: 128}}}, benchmark: {env: {CONC: '64'}}} +override_tp8_ep1_c80: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 64, max-running-requests: 160}}}, benchmark: {env: {CONC: '80'}}} +override_tp8_ep8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 8, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 4, max-running-requests: 8}}}, benchmark: {env: {CONC: '4'}}} +override_tp8_ep8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 8, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 16, max-running-requests: 32}}}, benchmark: {env: {CONC: '16'}}} +override_tp8_ep8_c24: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 8, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 24, max-running-requests: 48}}}, benchmark: {env: {CONC: '24'}}} +override_tp8_ep8_c32: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 8, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 32, max-running-requests: 64}}}, benchmark: {env: {CONC: '32'}}} +override_tp8_ep8_c40: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 8, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 40, max-running-requests: 80}}}, benchmark: {env: {CONC: '40'}}} +override_tp8_ep8_c48: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 8, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 48, max-running-requests: 96}}}, benchmark: {env: {CONC: '48'}}} +override_tp8_ep8_c64: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 8, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 64, max-running-requests: 128}}}, benchmark: {env: {CONC: '64'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 201631f2bc..7f4c7d99da 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -235,11 +235,11 @@ qwen3.5-fp8-mi325x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 6] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 32, 40, 48] } - - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 32, 40, 48] } - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 24, 32, 40, 48, 64, 80] } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 24, 32, 40, 48, 64] } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 6], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 32, 40, 48], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml } + - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 32, 40, 48], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 24, 32, 40, 48, 64, 80], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 24, 32, 40, 48, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml } qwen3.5-fp8-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 From b4931b8d973e2dbc4d0d37b83379b3ac7f948e9f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:22:29 -0500 Subject: [PATCH 016/105] feat(agentx): port the GLM-5.2 MI325X SGLang AgentX config to srt-slurm --- .../glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml | 75 +++++++++++++++++++ configs/amd-master.yaml | 2 +- 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..1bc8950316 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml @@ -0,0 +1,75 @@ +# GLM-5.2 FP8 AgentX on MI325X with SGLang native EAGLE MTP. The KV cache is +# GPU-resident. +base: + schema: 2 + name: glm5.2-fp8-mi325x-sglang-agentic + model: + path: hf:zai-org/GLM-5.2-FP8 + container: lmsysorg/sglang:v0.5.19-rocm720-mi30x + precision: fp8 + resources: + gpu_type: mi325x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 1 + dsa-prefill-backend: tilelang + dsa-decode-backend: tilelang + dsa-topk-backend: torch + kv-cache-dtype: bfloat16 + tool-call-parser: glm47 + reasoning-parser: glm45 + context-length: 1048576 + max-total-tokens: 1048576 + chunked-prefill-size: 131072 + mem-fraction-static: 0.85 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + env: + PYTHONNOUSERSITE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # The MI30X image's DSA top-k v2 JIT includes CUDA-only headers when + # compiled for gfx942. Use the portable Torch path, no fused top-k. + SGLANG_DSA_FUSE_TOPK: 'false' + SGLANG_OPT_USE_TOPK_V2: 'false' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: zai-org/GLM-5.2-FP8 + # GLM-5.2 natively supports 1M context, so replay the complete corpus. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + # Long responses admitted near the end of the measurement window can take + # minutes to finish; bound their drain without extending admission. + AIPERF_BENCHMARK_GRACE_PERIOD: '1800' + +# One variant per point. Admission and the decode graph batch are 2x CONC. +override_tp8_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 2}}}, benchmark: {env: {CONC: '1'}}} +override_tp8_c2: {roles: {agg: {args: {max-running-requests: 4, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '2'}}} +override_tp8_c3: {roles: {agg: {args: {max-running-requests: 6, cuda-graph-max-bs: 6}}}, benchmark: {env: {CONC: '3'}}} +override_tp8_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '4'}}} +override_tp8_c5: {roles: {agg: {args: {max-running-requests: 10, cuda-graph-max-bs: 10}}}, benchmark: {env: {CONC: '5'}}} +override_tp8_c6: {roles: {agg: {args: {max-running-requests: 12, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '6'}}} +override_tp8_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '8'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 7f4c7d99da..b8a5ef6057 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1061,7 +1061,7 @@ glm5.2-fp8-mi325x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 3, 4, 5, 6, 8] } + - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 3, 4, 5, 6, 8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml } minimaxm3-fp8-mi325x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:v0.27.1 From 28571426ef54d9dd8d7a2949c779ed99193d9959 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:23:38 -0500 Subject: [PATCH 017/105] feat(agentx): port the DSV4.1 Flash H100 SGLang AgentX configs to srt-slurm --- .../configs/dsv41flash-h100-block32.sh | 5 + .../sglang/h100-fp4-mtp/agentic.yaml | 97 +++++++++++++++++++ configs/nvidia-master.yaml | 4 +- 3 files changed, 104 insertions(+), 2 deletions(-) create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h100-block32.sh create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h100-block32.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h100-block32.sh new file mode 100755 index 0000000000..4332962a39 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h100-block32.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Install the measured H100 DeepSeek-V4.1-Flash block-32 tilings into the worker's SGLang. +set -euo pipefail +agentic=/infmax-workspace/benchmarks/single_node/agentic +python3 "$agentic/install_h100_block32_configs.py" "$agentic/kernel_configs/h100_dsv41_block32" /logs diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..ae23f7e092 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml @@ -0,0 +1,97 @@ +# DeepSeek-V4.1-Flash AgentX on H100 with SGLang native DSpark at TP8/EP8, as +# a plain TP8 curve and a DP8-attention sweep. H100 is not in the cookbook's +# hardware table: 80 GB cards cannot hold the resident weights plus the +# row-sharded Engram tables and still leave a KV pool, so the tables move to +# per-rank host shards and the prefill chunk is bounded so the sparse-attention +# indexer's [chunk, context] scoring buffer fits next to the weights at 1M +# context. The KV cache is GPU-resident. +# https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 +base: + schema: 2 + name: dsv41flash-fp4-h100-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40 + precision: fp4 + resources: + gpu_type: h100 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + # Hardware-specific tiling only; checkpoint data, scales and dtypes are unchanged. + setup_script: dsv41flash-h100-block32.sh + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + # Native MXFP4 Marlin supports Hopper with BF16 activations; dense FP8 + # operators and shipped DSpark precision remain unchanged. + moe-runner-backend: marlin + # 0.7 leaves 24 GB for the 4096-token chunk's working set: 0.8/4096 + # OOMed c8 (run 35304509605), and a 2048 chunk failed c4's latency + # coverage at ~1,000 prefill tok/s (run 35307250127). + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + # DSpark is the checkpoint's bundled draft; the block size is its only knob. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + # Bound long-prefill decode stalls. + prefill-decode-interval: 16 + reasoning-parser: auto + tool-call-parser: auto + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Long-context indexer masks change allocation sizes across requests. At + # C20 the stock allocator OOMed on a 2.39 GiB mask with 5.61 GiB reserved. + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV41_REASONING_EFFORT: high + # Row-sharded anonymous host Engram tables; the KV cache stays on GPU. + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC (at least TP under DP attention) +# with decode graphs clamped to 64-128. The TP SWA tail reserve is 32x CONC from +# c4 (8x below); the DP pool is per rank, and 64 tails/rank keeps the C16 TP +# aggregate 512-tail reserve. +override_tp8_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 8}}}, benchmark: {env: {CONC: '1'}}} +override_tp8_c2: {roles: {agg: {args: {max-running-requests: 4, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 16}}}, benchmark: {env: {CONC: '2'}}} +override_tp8_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 128}}}, benchmark: {env: {CONC: '4'}}} +override_tp8_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 256}}}, benchmark: {env: {CONC: '8'}}} +override_tp8_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 512}}}, benchmark: {env: {CONC: '16'}}} +override_tp8_c20: {roles: {agg: {args: {max-running-requests: 40, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 640}}}, benchmark: {env: {CONC: '20'}}} +# DP8 attention: the shipped MoE DSpark worker requires attn_tp=1. SGLang divides +# the engine-wide 4096-token chunk by DP (512 tokens/rank). The DP-aware router +# hashes AIPerf's per-session correlation ID so prefixes stay on one rank. +override_dp8_c4: {frontend: {type: sglang-router, args: {policy: consistent_hashing, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, max-running-requests: 8, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 64}}}, benchmark: {env: {CONC: '4', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'}}} +override_dp8_c8: {frontend: {type: sglang-router, args: {policy: consistent_hashing, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, max-running-requests: 16, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 64}}}, benchmark: {env: {CONC: '8', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'}}} +override_dp8_c16: {frontend: {type: sglang-router, args: {policy: consistent_hashing, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, max-running-requests: 32, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 64}}}, benchmark: {env: {CONC: '16', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'}}} +override_dp8_c20: {frontend: {type: sglang-router, args: {policy: consistent_hashing, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, max-running-requests: 40, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 64}}}, benchmark: {env: {CONC: '20', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 46179b44a9..e4cbad8a5d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8351,7 +8351,7 @@ dsv41flash-fp4-h100-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 20] } + - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml } # Sweep supported DP8 attention alongside the retained plain TP8 curve. dsv41flash-fp4-h100-sglang-agentic-dspark-dpa: @@ -8366,7 +8366,7 @@ dsv41flash-fp4-h100-sglang-agentic-dspark-dpa: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [4, 8, 16, 20] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [4, 8, 16, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml } dsv41flash-fp4-gb200-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 From 376cbc990d4de8e1a75eaa12ea551daa0b9e9909 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:23:38 -0500 Subject: [PATCH 018/105] feat(agentx): port the Qwen3.5 H100 SGLang AgentX configs to srt-slurm --- ...g-multi-tokenizer-cached-tokens-details.sh | 14 ++++ .../qwen3.5/sglang/h100-fp8-mtp/agentic.yaml | 78 +++++++++++++++++++ .../qwen3.5/sglang/h100-fp8/agentic.yaml | 75 ++++++++++++++++++ configs/nvidia-master.yaml | 8 +- 4 files changed, 171 insertions(+), 4 deletions(-) create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/sglang-multi-tokenizer-cached-tokens-details.sh create mode 100644 benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml create mode 100644 benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/sglang-multi-tokenizer-cached-tokens-details.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/sglang-multi-tokenizer-cached-tokens-details.sh new file mode 100755 index 0000000000..e0c0fb4a73 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/sglang-multi-tokenizer-cached-tokens-details.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Forward cached_tokens_details through SGLang's multi-tokenizer path, which +# older releases drop, so AIPerf sees per-request cache hits. +set -euo pipefail +file=/sgl-workspace/sglang/python/sglang/srt/managers/multi_tokenizer_mixin.py +if ! sed -n '/elif isinstance(output, BatchStrOutput):/,/input_token_logprobs_val=_extract_field_by_index/p' "$file" \ + | grep -q 'cached_tokens_details=_extract_field_by_index'; then + sed -i '/elif isinstance(output, BatchStrOutput):/,/input_token_logprobs_val=_extract_field_by_index/ { + /cached_tokens=_extract_field_by_index(output, "cached_tokens", i),/a\ + cached_tokens_details=_extract_field_by_index(\ + output, "cached_tokens_details", i\ + ), + }' "$file" +fi diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..bc42a7068f --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml @@ -0,0 +1,78 @@ +# Qwen3.5 FP8 AgentX on H100 with SGLang EAGLE MTP; attention is flashinfer +# (trtllm_mha is Blackwell-only). +base: + schema: 2 + name: qwen3.5-fp8-h100-sglang-agentic-mtp + model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + precision: fp8 + resources: + gpu_type: h100 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: flashinfer + enable-flashinfer-allreduce-fusion: true + # 80 GB HBM3 keeps the static fraction at 0.75. + mem-fraction-static: 0.75 + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + enable-metrics: true + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + env: + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_SPEC_V2: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + # H100 caps max_model_len at 131k; the 256k-capped with-subagents corpus + # keeps the rejection rate far below the unfiltered ~1M-token corpus. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_with_subagents_256k + +override_gpu_kv: + benchmark: + env: + KV_OFFLOADING: none +# HiCache extends RadixAttention. Hybrid GDN/Mamba allocates one KV and one +# Mamba host pool per rank: 1731 GB / TP8 / 2 pools = 108 GB each. +override_hicache: + roles: + agg: + args: + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 108 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml new file mode 100644 index 0000000000..6eeeaeca9d --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml @@ -0,0 +1,75 @@ +# Qwen3.5 FP8 AgentX on H100 with SGLang, no speculative decoding; attention is +# flashinfer (trtllm_mha is Blackwell-only). +base: + schema: 2 + name: qwen3.5-fp8-h100-sglang-agentic + model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang:v0.5.12-cu130 + precision: fp8 + resources: + gpu_type: h100 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + # This release drops cached_tokens_details on the multi-tokenizer path. + setup_script: sglang-multi-tokenizer-cached-tokens-details.sh + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: flashinfer + enable-flashinfer-allreduce-fusion: true + # 80 GB HBM3 keeps the static fraction at 0.75. + mem-fraction-static: 0.75 + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + # H100 caps max_model_len at 131k; the 256k-capped with-subagents corpus + # keeps the rejection rate far below the unfiltered ~1M-token corpus. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_with_subagents_256k + +override_gpu_kv: + benchmark: + env: + KV_OFFLOADING: none +# HiCache extends RadixAttention. Hybrid GDN/Mamba allocates one KV and one +# Mamba host pool per rank: 1731 GB / TP8 / 2 pools = 108 GB each. +override_hicache: + roles: + agg: + args: + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 108 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index e4cbad8a5d..d07e6af786 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5205,8 +5205,8 @@ qwen3.5-fp8-h100-sglang-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 14, 16] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [12, 14, 16, 20, 24, 28, 32, 42] } + - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 14, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml } + - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [12, 14, 16, 20, 24, 28, 32, 42], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml } # MTP speculative-decoding (spec-decoding: mtp) variant of # qwen3.5-fp8-h100-sglang-agentic: same TP8/EP8 GPU-resident and HiCache arms, @@ -5289,8 +5289,8 @@ qwen3.5-fp8-h100-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml } # Qwen3.8-Flash-Next FP8 AgentX on H100 via SGLang with native NEXTN MTP. # Day-zero recipe. H100 is Hopper, so FP8: NVFP4 needs SM100 tensor cores. The From 428972e0bc329759b683299b2005565f7fc15997 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:23:38 -0500 Subject: [PATCH 019/105] feat(agentx): port the Qwen3.5 H200 SGLang HiCache EP1 AgentX config to srt-slurm --- .../qwen3.5/sglang/h200-fp8-mtp/agentic.yaml | 18 ++++++++++++++++++ configs/nvidia-master.yaml | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml index 366f501006..519e6efafc 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml @@ -75,3 +75,21 @@ override_hicache: env: KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '1234' +# The same HiCache arm with the experts TP-sharded (EP1) on a later nightly. +override_hicache_ep1: + model: + container: lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9 + roles: + agg: + args: + expert-parallel-size: 1 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 77 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1234' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index d07e6af786..6e42526a27 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5275,7 +5275,7 @@ qwen3.5-fp8-h200-sglang-agentic-hicache-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [2, 4, 8, 10, 12, 16, 20, 24] } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [2, 4, 8, 10, 12, 16, 20, 24], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml } qwen3.5-fp8-h100-sglang-agentic-mtp: image: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 From e655a9570a3531c6c97a7b33f6bf87b51b357e4d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:24:46 -0500 Subject: [PATCH 020/105] feat(srt): accept the draft_model label for single-node DSpark points --- infx/srt_slurm/single_node.py | 3 +++ utils/test_srt_single_node.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/infx/srt_slurm/single_node.py b/infx/srt_slurm/single_node.py index 8a2c1dcae3..c9af2805c6 100644 --- a/infx/srt_slurm/single_node.py +++ b/infx/srt_slurm/single_node.py @@ -91,6 +91,9 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N if spec and spec["method"] not in {"eagle", "nextn", "mtp", "dspark"}: raise ValueError("Single-node SRT supports only native MTP, DSpark or no speculation") speculation = "mtp" if spec else "none" + # Some DSpark configs label the checkpoint's bundled draft as draft_model. + if spec and spec["method"] == "dspark" and environment["SPEC_DECODING"] == "draft_model": + speculation = "draft_model" agentic = environment["IS_AGENTIC"] == "1" expected = { "engine": (engine, SINGLE_NODE_ENGINES[environment["FRAMEWORK"]]), diff --git a/utils/test_srt_single_node.py b/utils/test_srt_single_node.py index 3717668320..55011e955d 100644 --- a/utils/test_srt_single_node.py +++ b/utils/test_srt_single_node.py @@ -125,6 +125,21 @@ def test_mtp_binding_uses_real_verification_and_preserves_expert_parallelism(poi runtime_arguments(f"{path}:base", env) +def test_dspark_accepts_draft_model_label_only_for_dspark(point): + path, recipe, env = point + recipe["roles"]["agg"]["args"].update({ + "speculative-algorithm": "DSPARK", "speculative-dspark-block-size": 6, + }) + recipe["benchmark"]["env"]["USE_CHAT_TEMPLATE"] = "true" + path.write_text(yaml.safe_dump({"base": recipe})) + runtime_arguments(f"{path}:base", {**env, "SPEC_DECODING": "draft_model"}) + recipe["roles"]["agg"]["args"]["speculative-algorithm"] = "EAGLE" + recipe["roles"]["agg"]["args"]["speculative-num-steps"] = 3 + path.write_text(yaml.safe_dump({"base": recipe})) + with pytest.raises(ValueError, match="SPEC_DECODING"): + runtime_arguments(f"{path}:base", {**env, "SPEC_DECODING": "draft_model"}) + + def test_concurrency_selector_keeps_graph_capture_coupled_to_client(point): path, recipe, env = point path.write_text(yaml.safe_dump({"base": recipe, "zip_override_conc": { From 72a734ff2329cda01e90209bc188af70dd6e96a3 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:25:42 -0500 Subject: [PATCH 021/105] feat(agentx): port the DSV4.1 Flash MI300X vLLM AgentX config to srt-slurm --- .../vllm/mi300x-fp4-mtp/agentic.yaml | 95 +++++++++++++++++++ configs/amd-master.yaml | 4 +- 2 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..7d649adfaa --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml @@ -0,0 +1,95 @@ +# DeepSeek-V4.1-Flash AgentX on MI300X (gfx942) with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). The KV cache is +# GPU-resident; the Engram tables stay on GPU at TP8 and move to pinned host +# memory below TP8, where the checkpoint share already fills a 192 GB card. +base: + schema: 2 + name: dsv41flash-fp4-mi300x-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984 + precision: fp4 + resources: + gpu_type: mi300x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + # The upstream recipe's name resolves to the Triton MXFP4 W4A16 kernel + # on gfx942. + moe-backend: aiter + gpu-memory-utilization: 0.9 + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. Adaptive + # verification trims requests on device, which the ROCm DeepSeek-V4 + # indexer does not support. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + # Upstream picks 1024 on GPUs with >= 160 GiB, and 2x CONC starves + # subagent fan-out at low CONC. Graph capture covers 128 x (1 + 5 drafts). + max-num-seqs: 128 + max-cudagraph-capture-size: 1024 + # The sparse-attention indexer allocates a [batched-tokens, 1M] fp8 + # logits buffer at startup. + max-num-batched-tokens: 8192 + # The ROCm sparse SWA path lacks the replay window clamp; bounded replay + # memory-faulted at the first prefix hit with a replay start. + no-swa-bounded-replay: true + # Piecewise capture segfaulted on gfx942; capture only full decode + # graphs and run prefill eagerly. + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + disable-uvicorn-access-log: true + env: + VLLM_ROCM_USE_AITER: '1' + VLLM_ROCM_USE_AITER_MOE: '1' + AITER_TRITON_LOG_LEVEL: ERROR + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + OMP_NUM_THREADS: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Full-context corpus for this 1M-context recipe. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. There is no TP2 arm: a 192 GB card measured negative KV +# memory at TP2 (run 35671005506). +override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '1'}}} +override_tp8_c2: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '2'}}} +override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '4'}}} +override_tp8_c8: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '8'}}} +override_tp8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '16'}}} +override_tp8_c32: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '32'}}} +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '1'}}} +override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '2'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '4'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '8'}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '16'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '32'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b8a5ef6057..0acaa942d2 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1036,10 +1036,10 @@ dsv41flash-fp4-mi300x-vllm-agentic-dspark: # the tables would not fit beside the checkpoint share on a 192 GB card. # vllm-project/vllm#57491 widened the two is_cuda() gates to # is_cuda_alike(), so engram_config resolves on gfx942 in this image. - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml } # TP4 halves the GPUs per server. With the Engram tables in pinned host # memory the resident weights are ~81 GiB per rank. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml } # No TP2 arm on this SKU. Measured in run 35671005506: at TP2 the resident # weights are ~145 GiB per rank, and even with the indexer buffer halved # to 4096 batched tokens the engine reported "Available KV cache memory: From 4331a36da6426f9bf247685ee0e0ac0e1f230cd2 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:25:42 -0500 Subject: [PATCH 022/105] feat(agentx): port the DSV4.1 Flash MI325X vLLM AgentX config to srt-slurm --- .../vllm/mi325x-fp4-mtp/agentic.yaml | 102 ++++++++++++++++++ configs/amd-master.yaml | 6 +- 2 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..1d2c034422 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml @@ -0,0 +1,102 @@ +# DeepSeek-V4.1-Flash AgentX on MI325X (gfx942) with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). The KV cache is +# GPU-resident; the Engram tables stay on GPU at TP8 and move to pinned host +# memory below TP8, where the checkpoint share already fills a 256 GB card. +base: + schema: 2 + name: dsv41flash-fp4-mi325x-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984 + precision: fp4 + resources: + gpu_type: mi325x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + # The upstream recipe's name resolves to the Triton MXFP4 W4A16 kernel + # on gfx942. + moe-backend: aiter + gpu-memory-utilization: 0.9 + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. Adaptive + # verification trims requests on device, which the ROCm DeepSeek-V4 + # indexer does not support. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + # Upstream picks 1024 on GPUs with >= 160 GiB, and 2x CONC starves + # subagent fan-out at low CONC. Graph capture covers 128 x (1 + 5 drafts). + max-num-seqs: 128 + max-cudagraph-capture-size: 1024 + # The sparse-attention indexer allocates a [batched-tokens, 1M] fp8 + # logits buffer at startup. + max-num-batched-tokens: 16384 + # The ROCm sparse SWA path lacks the replay window clamp; bounded replay + # memory-faulted at the first prefix hit with a replay start. + no-swa-bounded-replay: true + # Piecewise capture segfaulted on gfx942; capture only full decode + # graphs and run prefill eagerly. + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + disable-uvicorn-access-log: true + env: + VLLM_ROCM_USE_AITER: '1' + VLLM_ROCM_USE_AITER_MOE: '1' + AITER_TRITON_LOG_LEVEL: ERROR + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + OMP_NUM_THREADS: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Full-context corpus for this 1M-context recipe. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. TP2 leaves ~145 GiB of offloaded weights per rank, so it +# halves the indexer buffer, bounds the scheduler at 2x CONC within [16, 256] +# and stops capturing above 512 tokens. +override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '1'}}} +override_tp8_c2: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '2'}}} +override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '4'}}} +override_tp8_c8: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '8'}}} +override_tp8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '16'}}} +override_tp8_c32: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '32'}}} +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '1'}}} +override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '2'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '4'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '8'}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '16'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '32'}}} +override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-num-batched-tokens: 8192, max-num-seqs: 16, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '1'}}} +override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-num-batched-tokens: 8192, max-num-seqs: 16, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '2'}}} +override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-num-batched-tokens: 8192, max-num-seqs: 16, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '4'}}} +override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-num-batched-tokens: 8192, max-num-seqs: 16, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '8'}}} +override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-num-batched-tokens: 8192, max-num-seqs: 32, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '16'}}} +override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-num-batched-tokens: 8192, max-num-seqs: 64, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '32'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0acaa942d2..97e97a5049 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1101,13 +1101,13 @@ dsv41flash-fp4-mi325x-vllm-agentic-dspark: # the tables would not fit beside the checkpoint share on a 256 GB card. # vllm-project/vllm#57491 widened the two is_cuda() gates to # is_cuda_alike(), so engram_config resolves on gfx942 in this image. - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml } # TP4 halves the GPUs per server. With the Engram tables in pinned host # memory the resident weights are ~81 GiB per rank. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml } # TP2 quarters them. Resident weights rise to ~145 GiB per rank, so the # recipe halves the indexer buffer and caps the scheduler here. - - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml } dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260923 From fd37fe5e01c7cee39979d9d5e603013caf8cdbdf Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:26:02 -0500 Subject: [PATCH 023/105] fix(srt): bind single-node draft_model and EAGLE3 AgentX points The DSV4 B300 DSpark config labels its points spec-decoding: draft_model and the MiniMax-M3 TRT-LLM config drafts with EAGLE3; accept both as speculative recipes. Golden acceptance already resolves both curves. --- infx/srt_slurm/single_node.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infx/srt_slurm/single_node.py b/infx/srt_slurm/single_node.py index 8a2c1dcae3..ea529d0f8a 100644 --- a/infx/srt_slurm/single_node.py +++ b/infx/srt_slurm/single_node.py @@ -88,9 +88,8 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N if environment["FRAMEWORK"] not in {"sglang", "trt", "atom", "vllm"}: raise ValueError(f"Unsupported single-node framework: {environment['FRAMEWORK']!r}") spec = spec_parameters(role, engine) - if spec and spec["method"] not in {"eagle", "nextn", "mtp", "dspark"}: - raise ValueError("Single-node SRT supports only native MTP, DSpark or no speculation") - speculation = "mtp" if spec else "none" + if spec and spec["method"] not in {"eagle", "eagle3", "nextn", "mtp", "dspark"}: + raise ValueError("Single-node SRT supports only MTP, EAGLE3, DSpark or no speculation") agentic = environment["IS_AGENTIC"] == "1" expected = { "engine": (engine, SINGLE_NODE_ENGINES[environment["FRAMEWORK"]]), @@ -104,7 +103,8 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N "roles": (set(recipe["roles"]), {"agg"}), "benchmark type": (benchmark["type"], "custom"), "benchmark MODEL": (workload["MODEL"], environment["MODEL"]), - "SPEC_DECODING": (speculation, environment["SPEC_DECODING"]), + # The matrix labels speculation mtp or draft_model; either binds a speculative recipe. + "SPEC_DECODING": (bool(spec), environment["SPEC_DECODING"] != "none"), "AgentX client": (benchmark.get("command", "").endswith("srt_agentic.sh"), agentic), } if not agentic: From 6d086e6989f60745b156966f9e8575e392c6ec0a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:27:00 -0500 Subject: [PATCH 024/105] feat(srt): accept vLLM EAGLE3 speculation in the single-node adapter --- infx/srt_slurm/single_node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infx/srt_slurm/single_node.py b/infx/srt_slurm/single_node.py index 8a2c1dcae3..f921cb3c1c 100644 --- a/infx/srt_slurm/single_node.py +++ b/infx/srt_slurm/single_node.py @@ -88,8 +88,8 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N if environment["FRAMEWORK"] not in {"sglang", "trt", "atom", "vllm"}: raise ValueError(f"Unsupported single-node framework: {environment['FRAMEWORK']!r}") spec = spec_parameters(role, engine) - if spec and spec["method"] not in {"eagle", "nextn", "mtp", "dspark"}: - raise ValueError("Single-node SRT supports only native MTP, DSpark or no speculation") + if spec and spec["method"] not in {"eagle", "eagle3", "nextn", "mtp", "dspark"}: + raise ValueError("Single-node SRT supports only native MTP, EAGLE3, DSpark or no speculation") speculation = "mtp" if spec else "none" agentic = environment["IS_AGENTIC"] == "1" expected = { From aa8d1828cb3392d5c6d2be66c32e4dc8cb135845 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:27:30 -0500 Subject: [PATCH 025/105] feat(srt): accept EAGLE3 speculation in the single-node adapter MiniMax-M3 vLLM AgentX recipes speculate with the EAGLE3 GQA draft, which the golden acceptance lookup already maps to minimaxm3_eagle3_gqa. --- infx/srt_slurm/single_node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infx/srt_slurm/single_node.py b/infx/srt_slurm/single_node.py index 8a2c1dcae3..f921cb3c1c 100644 --- a/infx/srt_slurm/single_node.py +++ b/infx/srt_slurm/single_node.py @@ -88,8 +88,8 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N if environment["FRAMEWORK"] not in {"sglang", "trt", "atom", "vllm"}: raise ValueError(f"Unsupported single-node framework: {environment['FRAMEWORK']!r}") spec = spec_parameters(role, engine) - if spec and spec["method"] not in {"eagle", "nextn", "mtp", "dspark"}: - raise ValueError("Single-node SRT supports only native MTP, DSpark or no speculation") + if spec and spec["method"] not in {"eagle", "eagle3", "nextn", "mtp", "dspark"}: + raise ValueError("Single-node SRT supports only native MTP, EAGLE3, DSpark or no speculation") speculation = "mtp" if spec else "none" agentic = environment["IS_AGENTIC"] == "1" expected = { From 3266a317b10de044ae00640f033b5310b63f5830 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:27:30 -0500 Subject: [PATCH 026/105] feat(srt): accept EAGLE3 speculation in the single-node adapter MiniMax-M3 vLLM AgentX recipes speculate with the EAGLE3 GQA draft, which the golden acceptance lookup already maps to minimaxm3_eagle3_gqa. --- infx/srt_slurm/single_node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infx/srt_slurm/single_node.py b/infx/srt_slurm/single_node.py index 8a2c1dcae3..f921cb3c1c 100644 --- a/infx/srt_slurm/single_node.py +++ b/infx/srt_slurm/single_node.py @@ -88,8 +88,8 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N if environment["FRAMEWORK"] not in {"sglang", "trt", "atom", "vllm"}: raise ValueError(f"Unsupported single-node framework: {environment['FRAMEWORK']!r}") spec = spec_parameters(role, engine) - if spec and spec["method"] not in {"eagle", "nextn", "mtp", "dspark"}: - raise ValueError("Single-node SRT supports only native MTP, DSpark or no speculation") + if spec and spec["method"] not in {"eagle", "eagle3", "nextn", "mtp", "dspark"}: + raise ValueError("Single-node SRT supports only native MTP, EAGLE3, DSpark or no speculation") speculation = "mtp" if spec else "none" agentic = environment["IS_AGENTIC"] == "1" expected = { From fd82c451e6d654348847e381915b7b42e48ec426 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:27:52 -0500 Subject: [PATCH 027/105] feat(agentx): port the MiniMax-M3 MI325X vLLM AgentX config to srt-slurm --- .../vllm/mi325x-fp8-mtp/agentic.yaml | 74 +++++++++++++++++++ configs/amd-master.yaml | 2 +- 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..c4f69ae81b --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml @@ -0,0 +1,74 @@ +# MiniMax-M3 MXFP8 AgentX on MI325X with vLLM EAGLE3 (GQA draft). The KV cache +# is GPU-resident. +base: + schema: 2 + name: minimaxm3-fp8-mi325x-vllm-agentic + model: + path: hf:MiniMaxAI/MiniMax-M3-MXFP8 + container: vllm/vllm-openai-rocm:v0.27.1 + precision: fp8 + resources: + gpu_type: mi325x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: MiniMaxAI/MiniMax-M3-MXFP8 + tensor-parallel-size: 8 + gpu-memory-utilization: 0.90 + kv-cache-dtype: fp8 + block-size: 128 + language-model-only: true + attention-backend: TRITON_ATTN + enable-prefix-caching: true + enable-chunked-prefill: true + max-num-batched-tokens: 32768 + # Three-token EAGLE3; throughput runs replace verification with the + # golden acceptance length. + speculative-config: '{"method": "eagle3", "model": "Inferact/MiniMax-M3-EAGLE3-GQA", "num_speculative_tokens": 3, "attention_backend": "TRITON_ATTN"}' + tool-call-parser: minimax_m3 + reasoning-parser: minimax_m3 + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + trust-remote-code: true + stream-interval: 20 + env: + PYTHONNOUSERSITE: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: MiniMaxAI/MiniMax-M3-MXFP8 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. Admission is 2x CONC. +override_tp8_c1: {roles: {agg: {args: {max-num-seqs: 2}}}, benchmark: {env: {CONC: '1'}}} +override_tp8_c2: {roles: {agg: {args: {max-num-seqs: 4}}}, benchmark: {env: {CONC: '2'}}} +override_tp8_c4: {roles: {agg: {args: {max-num-seqs: 8}}}, benchmark: {env: {CONC: '4'}}} +override_tp8_c8: {roles: {agg: {args: {max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} +override_tp8_c10: {roles: {agg: {args: {max-num-seqs: 20}}}, benchmark: {env: {CONC: '10'}}} +override_tp8_c12: {roles: {agg: {args: {max-num-seqs: 24}}}, benchmark: {env: {CONC: '12'}}} +override_tp8_c14: {roles: {agg: {args: {max-num-seqs: 28}}}, benchmark: {env: {CONC: '14'}}} +override_tp8_c16: {roles: {agg: {args: {max-num-seqs: 32}}}, benchmark: {env: {CONC: '16'}}} +override_tp8_c18: {roles: {agg: {args: {max-num-seqs: 36}}}, benchmark: {env: {CONC: '18'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 97e97a5049..56a9ae3f57 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1075,7 +1075,7 @@ minimaxm3-fp8-mi325x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.20 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml } # DeepSeek-V4.1-Flash AgentX on MI325X (gfx942), a copy of the validated MI355X vLLM # arm (run 34710937012) on the same ROCm nightly. gfx942 is not in the upstream From 593906662ae6b3bfcec4dcdded1a962827b976e4 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:28:34 -0500 Subject: [PATCH 028/105] feat(agentx): port the Qwen3.5 FP4 B300 SGLang AgentX config to srt-slurm --- .../qwen3.5/sglang/b300-fp4-mtp/agentic.yaml | 99 +++++++++++++++++++ configs/nvidia-master.yaml | 6 +- 2 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..774646a201 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,99 @@ +# Qwen3.5-397B-A17B NVFP4 AgentX on B300 with SGLang native NEXTN MTP. +base: + schema: 2 + name: qwen3.5-fp4-b300-sglang-agentic + model: + path: hf:nvidia/Qwen3.5-397B-A17B-NVFP4 + container: lmsysorg/sglang:v0.5.16-cu130 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + trust-remote-code: true + data-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.8 + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-path: nvidia/Qwen3.5-397B-A17B-NVFP4 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-metrics: true + enable-cache-report: true + env: + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + # Outlast AIPerf's 300 s client connection pool. + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/Qwen3.5-397B-A17B-NVFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + # 256k-capped corpus, within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + +# One variant per point. Admission is 2x CONC and the graph batch is CONC, +# capped at 64. TP4 tokenizes with six workers to keep 256k warmups inside the +# client timeout; TP2 keeps SGLang's single tokenizer, which avoids a startup +# race with HiCache shared-memory initialization. HiCache sizes the target KV +# and Mamba host pools independently, and the NEXTN draft adds 1/15 of the +# target: (599 GB - 1 GB/rank) * 15 / TP2 / 31 = 144 GB per pool. +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: 'none'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: 'none'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: 'none'}}} +override_tp4_c12: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: 'none'}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 32, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: 'none'}}} +override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 40, cuda-graph-max-bs: 20}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: 'none'}}} +override_tp4_c24: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 48, cuda-graph-max-bs: 24}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: 'none'}}} +override_tp4_c28: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 56, cuda-graph-max-bs: 28}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: 'none'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 64, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: 'none'}}} +override_tp4_c40: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 80, cuda-graph-max-bs: 40}}}, benchmark: {env: {CONC: '40', KV_OFFLOADING: 'none'}}} +override_tp4_c48: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 96, cuda-graph-max-bs: 48}}}, benchmark: {env: {CONC: '48', KV_OFFLOADING: 'none'}}} +override_tp4_c56: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 112, cuda-graph-max-bs: 56}}}, benchmark: {env: {CONC: '56', KV_OFFLOADING: 'none'}}} +override_tp4_c60: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 120, cuda-graph-max-bs: 60}}}, benchmark: {env: {CONC: '60', KV_OFFLOADING: 'none'}}} +override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 128, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: 'none'}}} +override_tp4_c68: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 136, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '68', KV_OFFLOADING: 'none'}}} +override_tp4_c72: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 144, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '72', KV_OFFLOADING: 'none'}}} +override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: 'none'}}} +override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: 'none'}}} +override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: 'none'}}} +override_tp2_c12: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: 'none'}}} +override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 32, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: 'none'}}} +override_tp2_c20: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 40, cuda-graph-max-bs: 20}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: 'none'}}} +override_tp2_c24: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 48, cuda-graph-max-bs: 24}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: 'none'}}} +override_tp2_c28: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 56, cuda-graph-max-bs: 28}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: 'none'}}} +override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 64, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: 'none'}}} +override_tp2_c36_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 72, cuda-graph-max-bs: 36, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '36', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} +override_tp2_c44_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 88, cuda-graph-max-bs: 44, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '44', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} +override_tp2_c52_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 104, cuda-graph-max-bs: 52, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '52', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 46179b44a9..bea5313648 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5070,9 +5070,9 @@ qwen3.5-fp4-b300-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 64, 68, 72] } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32] } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [36, 44, 52] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 64, 68, 72], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [36, 44, 52], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml } # Qwen3.8-Flash-Next NVFP4 AgentX on B300 via SGLang with native NEXTN MTP. # Day-zero recipe; mirrors the B200 arm. TP1 per the cookbook's verified From fa3b4815236487b9587ecca88e66cb8f672f33e1 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:28:34 -0500 Subject: [PATCH 029/105] feat(agentx): port the Qwen3.5 FP8 B300 SGLang AgentX config to srt-slurm --- .../qwen3.5/sglang/b300-fp8-mtp/agentic.yaml | 104 ++++++++++++++++++ configs/nvidia-master.yaml | 6 +- 2 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..913dcba2fe --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml @@ -0,0 +1,104 @@ +# Qwen3.5-397B-A17B FP8 AgentX on B300 with SGLang native NEXTN MTP. +base: + schema: 2 + name: qwen3.5-fp8-b300-sglang-agentic + model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang:v0.5.16-cu130 + precision: fp8 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + data-parallel-size: 1 + enable-symm-mem: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.8 + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-path: Qwen/Qwen3.5-397B-A17B-FP8 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-metrics: true + enable-cache-report: true + env: + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + # Outlast AIPerf's 300 s client connection pool. + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + # 256k-capped corpus, within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + +# One variant per point. Admission is 2x CONC and the graph batch is CONC, +# capped at 64. TP4 tokenizes with six workers to keep 256k warmups inside the +# client timeout; TP2 keeps SGLang's single tokenizer, which avoids a startup +# race with HiCache shared-memory initialization. HiCache sizes the target KV +# and Mamba host pools independently, and the NEXTN draft adds 1/15 of the +# target: (599 GB - 1 GB/rank) * 15 / TP2 / 31 = 144 GB per pool. +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: 'none'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: 'none'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: 'none'}}} +override_tp4_c12: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: 'none'}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 32, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: 'none'}}} +override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 40, cuda-graph-max-bs: 20}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: 'none'}}} +override_tp4_c24: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 48, cuda-graph-max-bs: 24}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: 'none'}}} +override_tp4_c28: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 56, cuda-graph-max-bs: 28}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: 'none'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 64, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: 'none'}}} +override_tp4_c40: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 80, cuda-graph-max-bs: 40}}}, benchmark: {env: {CONC: '40', KV_OFFLOADING: 'none'}}} +override_tp4_c48: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 96, cuda-graph-max-bs: 48}}}, benchmark: {env: {CONC: '48', KV_OFFLOADING: 'none'}}} +override_tp4_c56: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 112, cuda-graph-max-bs: 56}}}, benchmark: {env: {CONC: '56', KV_OFFLOADING: 'none'}}} +override_tp4_c60: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 120, cuda-graph-max-bs: 60}}}, benchmark: {env: {CONC: '60', KV_OFFLOADING: 'none'}}} +override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 128, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: 'none'}}} +override_tp4_c68: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 136, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '68', KV_OFFLOADING: 'none'}}} +override_tp4_c72: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 144, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '72', KV_OFFLOADING: 'none'}}} +override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: 'none'}}} +override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: 'none'}}} +override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: 'none'}}} +override_tp2_c12: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: 'none'}}} +override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 32, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: 'none'}}} +override_tp2_c20: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 40, cuda-graph-max-bs: 20}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: 'none'}}} +override_tp2_c24: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 48, cuda-graph-max-bs: 24}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: 'none'}}} +override_tp2_c28: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 56, cuda-graph-max-bs: 28}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: 'none'}}} +override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 64, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: 'none'}}} +override_tp2_c32_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 64, cuda-graph-max-bs: 32, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} +override_tp2_c34_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 68, cuda-graph-max-bs: 34, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '34', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} +override_tp2_c36_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 72, cuda-graph-max-bs: 36, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '36', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} +override_tp2_c38_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 76, cuda-graph-max-bs: 38, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '38', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} +override_tp2_c40_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 80, cuda-graph-max-bs: 40, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '40', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} +override_tp2_c44_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 88, cuda-graph-max-bs: 44, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '44', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} +override_tp2_c48_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 96, cuda-graph-max-bs: 48, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '48', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} +override_tp2_c52_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 104, cuda-graph-max-bs: 52, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '52', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} +override_tp2_c56_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 112, cuda-graph-max-bs: 56, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '56', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index bea5313648..604b93e297 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5054,9 +5054,9 @@ qwen3.5-fp8-b300-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 64, 68, 72] } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32] } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 34, 36, 38, 40, 44, 48, 52, 56] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 64, 68, 72], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 34, 36, 38, 40, 44, 48, 52, 56], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml } qwen3.5-fp4-b300-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 From d8e5ba55221de1ebe2ae61575e200c9698498d07 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:28:35 -0500 Subject: [PATCH 030/105] feat(agentx): port the GLM-5.2 FP4 B300 SGLang AgentX config to srt-slurm --- .../glm5.2/sglang/b300-fp4-mtp/agentic.yaml | 83 +++++++++++++++++++ configs/nvidia-master.yaml | 2 +- 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..573ecd9fce --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,83 @@ +# GLM-5.2 NVFP4 AgentX on B300 with SGLang EAGLE off the checkpoint's nextn +# head: the low-latency TP8 arm (fp8 KV, cutedsl bf16 GEMM). HiCache spills +# evicted prefixes to host DRAM; without it the working set outgrows HBM past +# c8 and the radix hit rate collapses. +base: + schema: 2 + name: glm5.2-fp4-b300-sglang-agentic + model: + path: hf:nvidia/GLM-5.2-NVFP4 + container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 1 + kv-cache-dtype: fp8_e4m3 + bf16-gemm-backend: cutedsl + max-prefill-tokens: 8192 + quantization: modelopt_fp4 + # GLM-5.2 emits the GLM-4.7 tool-call format. + tool-call-parser: glm47 + reasoning-parser: glm45 + chunked-prefill-size: 8192 + mem-fraction-static: 0.85 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + # MLA-family: every TP rank holds complete per-token KV, so the host + # pool is an absolute 270 GB/rank; the DSA indexer pool rides on top. + enable-hierarchical-cache: true + hicache-size: 270 + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + watchdog-timeout: 1800 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/GLM-5.2-NVFP4 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2399' + +# One variant per point: admission is 2x CONC and the graph batch matches it, +# capped at 64. +override_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 2}}}, benchmark: {env: {CONC: '1'}}} +override_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '4'}}} +override_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '8'}}} +override_c12: {roles: {agg: {args: {max-running-requests: 24, cuda-graph-max-bs: 24}}}, benchmark: {env: {CONC: '12'}}} +override_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '16'}}} +override_c20: {roles: {agg: {args: {max-running-requests: 40, cuda-graph-max-bs: 40}}}, benchmark: {env: {CONC: '20'}}} +override_c24: {roles: {agg: {args: {max-running-requests: 48, cuda-graph-max-bs: 48}}}, benchmark: {env: {CONC: '24'}}} +override_c28: {roles: {agg: {args: {max-running-requests: 56, cuda-graph-max-bs: 56}}}, benchmark: {env: {CONC: '28'}}} +override_c32: {roles: {agg: {args: {max-running-requests: 64, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '32'}}} +override_c40: {roles: {agg: {args: {max-running-requests: 80, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '40'}}} +override_c48: {roles: {agg: {args: {max-running-requests: 96, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '48'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 604b93e297..1645f95600 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7095,7 +7095,7 @@ glm5.2-fp4-b300-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48], srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml } # First GLM-5.2 FP8 AgentX recipe on B300, the FP8 precision sibling of # glm5.2-fp4-b300-sglang-agentic-mtp and the B300 counterpart of From 8f258c54dc1f7999148467b9b34e4dc1d1adf955 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:28:35 -0500 Subject: [PATCH 031/105] feat(agentx): port the GLM-5.2 FP8 B300 SGLang AgentX config to srt-slurm --- .../glm5.2/sglang/b300-fp8-mtp/agentic.yaml | 79 +++++++++++++++++++ configs/nvidia-master.yaml | 2 +- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..b3b183dfab --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml @@ -0,0 +1,79 @@ +# GLM-5.2 FP8 AgentX on B300 with SGLang EAGLE off the checkpoint's nextn +# head: the low-latency TP8 arm (fp8 KV, cutedsl bf16 GEMM). HiCache spills +# evicted prefixes to host DRAM; without it the working set outgrows HBM past +# c8 and the radix hit rate collapses. +base: + schema: 2 + name: glm5.2-fp8-b300-sglang-agentic + model: + path: hf:zai-org/GLM-5.2-FP8 + container: lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b + precision: fp8 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 1 + kv-cache-dtype: fp8_e4m3 + bf16-gemm-backend: cutedsl + max-prefill-tokens: 8192 + quantization: fp8 + # GLM-5.2 emits the GLM-4.7 tool-call format. + tool-call-parser: glm47 + reasoning-parser: glm45 + chunked-prefill-size: 8192 + mem-fraction-static: 0.85 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + # MLA-family: every TP rank holds complete per-token KV, so the host + # pool is an absolute 270 GB/rank; the DSA indexer pool rides on top. + enable-hierarchical-cache: true + hicache-size: 270 + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + watchdog-timeout: 1800 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # Keep FlashInfer, Triton and JIT caches allocation-local. + SGLANG_CACHE_DIR: /tmp/sglang-cache + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: zai-org/GLM-5.2-FP8 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2399' + +# One variant per point: admission is 2x CONC and the graph batch matches it, +# capped at 64. +override_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 2}}}, benchmark: {env: {CONC: '1'}}} +override_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '4'}}} +override_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '8'}}} +override_c12: {roles: {agg: {args: {max-running-requests: 24, cuda-graph-max-bs: 24}}}, benchmark: {env: {CONC: '12'}}} +override_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '16'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 1645f95600..3d458466b8 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7126,7 +7126,7 @@ glm5.2-fp8-b300-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml } # First GLM-5.2 AgentX recipe on B200, shipped spec-decode-only per the AgentX # policy that agentic arms enable speculative decoding rather than running a From 931b3c6850a4909dc50d610fa8973a6413efccee Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:28:35 -0500 Subject: [PATCH 032/105] feat(agentx): port the DSV4 Pro B300 SGLang AgentX config to srt-slurm --- .../dsv4/sglang/b300-fp4-mtp/agentic.yaml | 148 ++++++++++++++++++ configs/nvidia-master.yaml | 4 +- 2 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..c7618db209 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,148 @@ +# DeepSeek-V4-Pro-0813 AgentX on B300 with SGLang DSpark (block 6). Two arms: +# TP8 with the flashinfer MXFP4 MoE and GPU-resident KV, and DEP8 (attention +# DP + Mega-MoE + FP4 indexer) behind the SGLang router with a HiCache DRAM tier. +base: + schema: 2 + name: dsv4-fp4-b300-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 + container: lmsysorg/sglang:nightly-dev-20260901-07c8f729 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tensor-parallel-size: 8 + moe-runner-backend: flashinfer_mxfp4 + disable-flashinfer-autotune: true + mem-fraction-static: 0.88 + swa-full-tokens-ratio: 0.1 + allow-auto-truncate: true + chunked-prefill-size: 8192 + tool-call-parser: deepseekv4 + reasoning-parser: deepseek-v4 + chat-template: /infmax-workspace/benchmarks/single_node/chat_templates/deepseek_v4_thinking.jinja + watchdog-timeout: 1800 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + attention-backend: compressed + page-size: 256 + disable-shared-experts-fusion: true + enable-metrics: true + enable-cache-report: true + env: + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: '1' + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_USE_JIT_NORM: '1' + SGLANG_OPT_USE_JIT_INDEXER_METADATA: '1' + SGLANG_OPT_USE_TOPK_V2: '1' + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + # Triton compiles with the image's CUDA ptxas. + TRITON_PTXAS_PATH: /usr/local/cuda/bin/ptxas + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. TP8: admission is 2x CONC and the graph batch 4x +# CONC, capped at 64. +override_tp8_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp8_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp8_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} +override_tp8_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: none}}} +override_tp8_c32: {roles: {agg: {args: {max-running-requests: 64, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: none}}} + +# DEP8 replaces the TP8 MoE path: the router keeps each session on the DP rank +# holding its radix prefix, the global prefill chunk is 8192 per rank, decode +# graphs cover the padded speculative batch across ranks, and AIPerf owns the +# warmup. HiCache capacity is a host/device ratio; 3 keeps the tier near 2 TB. +# Mega-MoE's transient workspace sits outside the static pool, so the static +# fraction steps down with CONC: 0.90 to c256, 0.88 at c384, 0.86 from c512. +override_dep8_c32: + frontend: &dep8_frontend + type: sglang-router + args: + policy: consistent_hashing + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + # A transient router-to-engine send failure would otherwise abort the run. + retry-max-retries: 8 + retry-initial-backoff-ms: 500 + retry-max-backoff-ms: 10000 + retry-backoff-multiplier: 2 + roles: + agg: + args: + <<: &dep8_args + moe-runner-backend: null + data-parallel-size: 8 + tokenizer-worker-num: 8 + enable-prefill-delayer: true + prefill-decode-interval: 20 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + incremental-streaming-output: true + stream-interval: 20 + expert-parallel-size: 8 + moe-a2a-backend: megamoe + enable-w4a4-mxfp4-megamoe: true + enable-deepseek-v4-fp4-indexer: true + chunked-prefill-size: 65536 + cuda-graph-max-bs-decode: 544 + swa-full-tokens-ratio: 0.075 + enable-hierarchical-cache: true + hicache-ratio: 3 + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + skip-server-warmup: true + max-running-requests: 64 + mem-fraction-static: 0.9 + env: &dep8_env + # Covers the 8192-token per-rank prefill budget. + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320' + benchmark: + env: + <<: &dep8_client + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2849' + CONC: '32' +override_dep8_c64: {frontend: *dep8_frontend, roles: {agg: {args: {<<: *dep8_args, max-running-requests: 128, mem-fraction-static: 0.9}, env: *dep8_env}}, benchmark: {env: {<<: *dep8_client, CONC: '64'}}} +override_dep8_c128: {frontend: *dep8_frontend, roles: {agg: {args: {<<: *dep8_args, max-running-requests: 256, mem-fraction-static: 0.9}, env: *dep8_env}}, benchmark: {env: {<<: *dep8_client, CONC: '128'}}} +override_dep8_c256: {frontend: *dep8_frontend, roles: {agg: {args: {<<: *dep8_args, max-running-requests: 512, mem-fraction-static: 0.9}, env: *dep8_env}}, benchmark: {env: {<<: *dep8_client, CONC: '256'}}} +override_dep8_c384: {frontend: *dep8_frontend, roles: {agg: {args: {<<: *dep8_args, max-running-requests: 768, mem-fraction-static: 0.88}, env: *dep8_env}}, benchmark: {env: {<<: *dep8_client, CONC: '384'}}} +override_dep8_c512: {frontend: *dep8_frontend, roles: {agg: {args: {<<: *dep8_args, max-running-requests: 1024, mem-fraction-static: 0.86}, env: *dep8_env}}, benchmark: {env: {<<: *dep8_client, CONC: '512'}}} +override_dep8_c576: {frontend: *dep8_frontend, roles: {agg: {args: {<<: *dep8_args, max-running-requests: 1152, mem-fraction-static: 0.86}, env: *dep8_env}}, benchmark: {env: {<<: *dep8_client, CONC: '576'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3d458466b8..8a6c885a9a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1130,8 +1130,8 @@ dsv4-fp4-b300-sglang-agentic-hicache-mtp: agentic-coding: - dram-utilization: 0.95 search-space: - - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [32, 64, 128, 256, 384, 512, 576], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [32, 64, 128, 256, 384, 512, 576], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml } # DeepSeek-V4-Pro-0813 on B300 with DSpark speculative decoding. Recipe is # selected inside benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh by From 20d3dd19e81fb1eb7e75d12f2abc61bca1dbab5c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:28:35 -0500 Subject: [PATCH 033/105] feat(agentx): port the MiniMax-M3 B300 TRT-LLM AgentX config to srt-slurm --- .../configs/minimaxm3-trtllm-agentx.sh | 17 +++ .../trtllm/b300-fp4-mtp/agentic.yaml | 113 ++++++++++++++++++ configs/nvidia-master.yaml | 6 +- 3 files changed, 133 insertions(+), 3 deletions(-) create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-trtllm-agentx.sh create mode 100644 benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-trtllm-agentx.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-trtllm-agentx.sh new file mode 100755 index 0000000000..d84e9c2787 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-trtllm-agentx.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Prepare a TensorRT-LLM 1.3 worker for MiniMax-M3 AgentX: keep request and +# iteration metrics without rc23's per-step timing collector, and accept +# OpenAI's store=false chat field. Every rank runs this; the lock serializes +# ranks that share a container, and each step is a no-op once applied. +set -euo pipefail +exec 9>/tmp/minimaxm3-trtllm-agentx.lock +flock 9 +root=$(python3 -c 'from importlib.util import find_spec; from pathlib import Path; print(Path(find_spec("tensorrt_llm").origin).parent)') +executor="$root/_torch/pyexecutor/py_executor.py" +gate="enabled=getattr(self.llm_args, 'return_perf_metrics', False))" +if grep -Fq "$gate" "$executor"; then + sed -i "s/enabled=getattr(self.llm_args, 'return_perf_metrics', False))/enabled=False)/" "$executor" +fi +grep -Fq "self.perf_manager = PerfMetricsManager(" "$executor" +grep -Fq "enabled=False)" "$executor" +python3 /infmax-workspace/runners/patch_trtllm_chat_store.py diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..6ede031d15 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,113 @@ +# MiniMax-M3 NVFP4 AgentX on B300 with TensorRT-LLM EAGLE3-GQA. DRAM KV offload +# uses TRT-LLM's native secondary-memory pool, pinned per topology. +base: + schema: 2 + name: minimaxm3-fp4-b300-trtllm-agentic + model: + path: hf:nvidia/MiniMax-M3-NVFP4 + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: trtllm_serve + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: trtllm + served_model_name: nvidia/MiniMax-M3-NVFP4 + setup_script: minimaxm3-trtllm-agentx.sh + roles: + agg: + nodes: 1 + workers: 1 + extra_args: [--chat_template, /model/chat_template.jinja] + args: + moe_expert_parallel_size: 1 + max_seq_len: 1048576 + max_num_tokens: 16384 + cuda_graph_config: + enable_padding: true + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: [1, 512, 1024, 2048] + enable_userbuffers: true + max_num_streams: 3 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + free_gpu_memory_fraction: 0.94 + enable_block_reuse: true + block_reuse_policy: per_conversation + tokens_per_block: 128 + use_kv_cache_manager_v2: true + dtype: fp8 + event_buffer_max_size: 0 + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + enable_chunked_prefill: true + enable_autotuner: true + trust_remote_code: true + reasoning_parser: minimax_m3 + stream_interval: 20 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: true + num_postprocess_workers: 8 + enable_attention_dp: false + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TLLM_PROFILE_LOG_RANKS: all + PYTHONNOUSERSITE: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TQDM_DISABLE: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/MiniMax-M3-NVFP4 + # trtllm-serve exposes Prometheus text at /prometheus/metrics. + AIPERF_SERVER_METRICS_URLS: http://localhost:8000/prometheus/metrics + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + KV_OFFLOADING: dram + +# One variant per point. The batch limit is CONC with graphs captured at every +# size to 20 and a thinned ladder at 25 and 30. The host pool is 288 GiB at TP8 +# and about 362 GiB below. +override_tp2_c20: {roles: {agg: {gpus: 2, args: {tensor_parallel_size: 2, max_batch_size: 20, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '20', TOTAL_CPU_DRAM_GB: '599'}}} +override_tp2_c25: {roles: {agg: {gpus: 2, args: {tensor_parallel_size: 2, max_batch_size: 25, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 19, 21, 23, 25]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '25', TOTAL_CPU_DRAM_GB: '599'}}} +override_tp2_c30: {roles: {agg: {gpus: 2, args: {tensor_parallel_size: 2, max_batch_size: 30, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 18, 20, 22, 24, 27, 30]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '30', TOTAL_CPU_DRAM_GB: '599'}}} +override_tp4_c5: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 5, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '5', TOTAL_CPU_DRAM_GB: '1199'}}} +override_tp4_c10: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 10, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '10', TOTAL_CPU_DRAM_GB: '1199'}}} +override_tp4_c15: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 15, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '15', TOTAL_CPU_DRAM_GB: '1199'}}} +override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 20, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '20', TOTAL_CPU_DRAM_GB: '1199'}}} +override_tp4_c25: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 25, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 19, 21, 23, 25]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '25', TOTAL_CPU_DRAM_GB: '1199'}}} +override_tp4_c30: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 30, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 18, 20, 22, 24, 27, 30]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '30', TOTAL_CPU_DRAM_GB: '1199'}}} +override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor_parallel_size: 8, max_batch_size: 1, cuda_graph_config: {batch_sizes: [1]}, kv_cache_config: {host_cache_size: 309237645312}}}}, benchmark: {env: {CONC: '1', TOTAL_CPU_DRAM_GB: '2399'}}} +override_tp8_c5: {roles: {agg: {gpus: 8, args: {tensor_parallel_size: 8, max_batch_size: 5, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5]}, kv_cache_config: {host_cache_size: 309237645312}}}}, benchmark: {env: {CONC: '5', TOTAL_CPU_DRAM_GB: '2399'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 8a6c885a9a..ca65a27b0b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5672,9 +5672,9 @@ minimaxm3-fp4-b300-trtllm-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [20, 25, 30] } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [5, 10, 15, 20, 25, 30] } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5] } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [20, 25, 30], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [5, 10, 15, 20, 25, 30], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml } # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve # covers the middle/high-interactivity range omitted by the one-decode DEP From 05aa0898883ff1fad9ef532dadac833482d94dfc Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:28:35 -0500 Subject: [PATCH 034/105] fix(b300): serve Qwen3.8-Flash-Next NVFP4 natively from the writable model root The checkpoint is not staged on node-local NVMe; the legacy AgentX path downloads it to the shared writable model directory. --- runners/launch_b300-dsxe.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index ba0269b9ca..92c20ec44a 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -139,6 +139,10 @@ if [[ "$EXECUTION_PATH" == native-single-node ]]; then if [[ "$MODEL" == nvidia/DeepSeek-R1-0528-FP4-V2 ]]; then SRT_MODEL_PATH="$MODEL_ROOT/DeepSeek-R1-0528-NVFP4-v2" fi + # Not staged on node-local NVMe; the legacy AgentX path downloaded it here. + if [[ "$MODEL" == RadixArk/Qwen3.8-Flash-Next-NVFP4 ]]; then + SRT_MODEL_PATH="$WRITABLE_MODELS_DIR/${MODEL##*/}" + fi SRT_SQUASH_FILE="$SQUASH_DIR/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" launch_srt_single_node b300-dsxe \ --var SLURM_ACCOUNT "$SLURM_ACCOUNT" --var SLURM_PARTITION "$SLURM_PARTITION" \ From ed2f0f047dc243b89aabbc7bef2c9e132a191a93 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:28:35 -0500 Subject: [PATCH 035/105] feat(agentx): port the Qwen3.8-Flash-Next FP4 B300 SGLang AgentX config to srt-slurm --- .../sglang/b300-fp4-mtp/agentic.yaml | 72 +++++++++++++++++++ configs/nvidia-master.yaml | 2 +- 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..5ba1dd8652 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,72 @@ +# Qwen3.8-Flash-Next NVFP4 AgentX on B300 via SGLang with native NEXTN MTP. +# The 126 GiB checkpoint fits on one B300, so the cookbook command is TP1. +base: + schema: 2 + name: qwen3.8next-fp4-b300-sglang-agentic + model: + path: hf:RadixArk/Qwen3.8-Flash-Next-NVFP4 + container: lmsysorg/sglang:qwen38flashnext + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 1 + args: + served-model-name: RadixArk/Qwen3.8-Flash-Next-NVFP4 + trust-remote-code: true + tensor-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + # Quantization is read from the checkpoint; the hybrid GDN + # linear-attention layers take their own backends. + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + # SM100+ flashinfer linear-attn decode requires a bf16 SSM state. + mamba-ssm-dtype: bfloat16 + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + reasoning-parser: auto + mem-fraction-static: 0.8 + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-path: RadixArk/Qwen3.8-Flash-Next-NVFP4 + enable-metrics: true + enable-cache-report: true + env: + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + # Outlast AIPerf's 300 s client connection pool. + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: RadixArk/Qwen3.8-Flash-Next-NVFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + # 256k-capped corpus, within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + +# One variant per point: admission is 2x CONC and the graph batch is CONC. NEXTN +# resets an unset admission limit to 48, so it stays explicit. +override_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1'}}} +override_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4'}}} +override_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8'}}} +override_c12: {roles: {agg: {args: {max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12'}}} +override_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '16'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index ca65a27b0b..b18bf9cebe 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5089,7 +5089,7 @@ qwen3.8next-fp4-b300-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } + - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml } # Controlled AgentX power A/B: identical software, topology, MTP settings, # concurrency, and memory tier across FP8 and FP4. The HBM-only rows measure # the natural AgentX prefix-cache workload; HiCache isolates host-tier effects. From a5ecd119ad84542d68d43dc514398763ff4b6505 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:21:55 -0500 Subject: [PATCH 036/105] feat(srt): accept draft_model speculation in the single-node adapter DeepSeek-V4 configs mark their bundled DSpark draft as draft_model; the recipe still speculates natively, so the point binds as speculative. --- infx/srt_slurm/single_node.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/infx/srt_slurm/single_node.py b/infx/srt_slurm/single_node.py index f921cb3c1c..9a97c66fe4 100644 --- a/infx/srt_slurm/single_node.py +++ b/infx/srt_slurm/single_node.py @@ -104,7 +104,11 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N "roles": (set(recipe["roles"]), {"agg"}), "benchmark type": (benchmark["type"], "custom"), "benchmark MODEL": (workload["MODEL"], environment["MODEL"]), - "SPEC_DECODING": (speculation, environment["SPEC_DECODING"]), + # draft_model names a bundled or separate draft; its recipes speculate natively. + "SPEC_DECODING": ( + speculation, + "mtp" if environment["SPEC_DECODING"] == "draft_model" else environment["SPEC_DECODING"], + ), "AgentX client": (benchmark.get("command", "").endswith("srt_agentic.sh"), agentic), } if not agentic: From 3f24127de8f3d08b9def79818af26f32e41b15d7 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:21:55 -0500 Subject: [PATCH 037/105] feat(srt): force ATOM AgentX golden acceptance with its server flag ATOM pins acceptance with --spec-decode-acceptance-length rather than an environment variable. The adapter now sets it from the golden curve for AgentX throughput and removes it for evals, reading the draft model for the MiniMax GQA curve and the probabilistic sampler for Kimi DSpark. --- infx/srt_slurm/synthetic_acceptance.py | 15 ++++++++++++++- utils/test_synthetic_acceptance.py | 22 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/infx/srt_slurm/synthetic_acceptance.py b/infx/srt_slurm/synthetic_acceptance.py index be7232eb12..724eb535d6 100644 --- a/infx/srt_slurm/synthetic_acceptance.py +++ b/infx/srt_slurm/synthetic_acceptance.py @@ -27,6 +27,7 @@ "dynamo-sglang": "sglang", "trt": "trtllm", "dynamo-trt": "trtllm", + "atom": "atom", } SGLANG_VARIABLES = ( "SGLANG_SIMULATE_ACC_LEN", @@ -42,10 +43,15 @@ def spec_parameters(role: Mapping[str, Any], engine: str) -> dict[str, Any]: method = args.get("method") if not method: return {} - return { + spec = { "method": str(method).lower(), "num_speculative_tokens": args.get("num-speculative-tokens"), + "model": args.get("draft-model", ""), } + if spec["method"] == "dspark": + # ATOM DSpark verifies with probabilistic block rejection sampling. + spec["draft_sample_method"] = "probabilistic" + return spec if engine == "vllm": raw = args.get("speculative-config") if raw is None: @@ -181,6 +187,13 @@ def build_overrides( "--set", f"{prefix}.args.speculative-config={json.dumps(worker_spec)}", ] + elif engine == "atom": + # ATOM forces acceptance with a server flag rather than environment. + key = "spec-decode-acceptance-length" + if al is not None and worker_spec: + overrides += ["--set", f"{prefix}.args.{key}={al:g}"] + elif key in (role.get("args") or {}): + overrides += ["--unset", f"{prefix}.args.{key}"] elif al is not None and worker_spec: values = ( (f"{al:g}", "match-expected", "real-draft-token") diff --git a/utils/test_synthetic_acceptance.py b/utils/test_synthetic_acceptance.py index ffa8b01aa0..a3ae8f92f2 100644 --- a/utils/test_synthetic_acceptance.py +++ b/utils/test_synthetic_acceptance.py @@ -267,6 +267,28 @@ def test_real_runs_clear_synthetic_without_a_curve( assert result["environment"]["KEEP"] == "yes" +@pytest.mark.parametrize( + ("prefix", "args", "expected"), + [ + ("dsv4", {"method": "dspark"}, 2.7), + ("kimik3", {"method": "dspark", "draft-model": "Inferact/Kimi-K3-DSpark"}, 2.9), + ("minimaxm3", {"method": "eagle3", "draft-model": "Inferact/MiniMax-M3-EAGLE3-GQA"}, 2.6), + ], +) +def test_atom_forces_golden_acceptance_by_server_flag( + golden_dir: Path, prefix: str, args: dict[str, Any], expected: float +) -> None: + recipe = {"roles": {"agg": {"args": {**args, "num-speculative-tokens": 3}, "env": {}}}} + env = {**ENV, "MODEL_PREFIX": prefix} + result = apply_native(recipe, build_overrides(recipe, "atom", env, golden_dir=golden_dir)) + assert result["roles"]["agg"]["args"]["spec-decode-acceptance-length"] == expected + # Evals verify real drafts, so a recipe-pinned acceptance length is removed. + evaluated = apply_native( + result, build_overrides(result, "atom", {**env, "EVAL_ONLY": "true"}, golden_dir=golden_dir) + ) + assert "spec-decode-acceptance-length" not in evaluated["roles"]["agg"]["args"] + + @pytest.mark.parametrize( "curve", [ From 84fdf52e9a3e733b61af122276bff64a295199be Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:22:22 -0500 Subject: [PATCH 038/105] fix(mi355x): mount the shared HF cache for native AgentX checkpoints The legacy MI355X AgentX path read DeepSeek-V4-Pro (vLLM/ATOM), DeepSeek-V4.1 Flash, MiniMax-M3 and GLM-5.2-FP8 from /it-share rather than node-local NVMe. The native srt-slurm path keeps that mount for those AgentX checkpoints. --- runners/launch_mi355x-amds.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index b3eec28469..38b15fe1a9 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -15,6 +15,17 @@ if [[ "$EXECUTION_PATH" == native-single-node ]]; then check_env_vars GITHUB_WORKSPACE MODEL IMAGE source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" || exit 1 export HF_HUB_CACHE_MOUNT=/var/lib/hf-hub-cache/ + # AgentX checkpoints the legacy scripts read from the shared NFS cache. + if [[ "$IS_AGENTIC" == 1 ]]; then + case "$MODEL" in + MiniMaxAI/MiniMax-M3*|amd/MiniMax-M3*|zai-org/GLM-5.2-FP8|deepseek-ai/DeepSeek-V4.1-Flash) + export HF_HUB_CACHE_MOUNT=/it-share/hf-hub-cache/ ;; + deepseek-ai/DeepSeek-V4-Pro|deepseek-ai/DeepSeek-V4-Pro-0813) + if [[ "$FRAMEWORK" == vllm || "$FRAMEWORK" == atom ]]; then + export HF_HUB_CACHE_MOUNT=/it-share/hf-hub-cache/ + fi ;; + esac + fi export SRT_MODEL_PATH="hf:$MODEL" export SALLOC_TIME_LIMIT=500 export SRT_SRUN_OPTIONS='{"container-remap-root":"", "container-writable":""}' From 6395aec7449d275f1e5dc34c24ddd49607d2e151 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:28:53 -0500 Subject: [PATCH 039/105] feat(srt): accept draft-model labeled speculation in the single-node adapter The DSV4 B200 vLLM AgentX matrix labels its native DSpark drafter draft_model rather than mtp. --- infx/srt_slurm/single_node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infx/srt_slurm/single_node.py b/infx/srt_slurm/single_node.py index f921cb3c1c..fcf8fef7ee 100644 --- a/infx/srt_slurm/single_node.py +++ b/infx/srt_slurm/single_node.py @@ -104,7 +104,8 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N "roles": (set(recipe["roles"]), {"agg"}), "benchmark type": (benchmark["type"], "custom"), "benchmark MODEL": (workload["MODEL"], environment["MODEL"]), - "SPEC_DECODING": (speculation, environment["SPEC_DECODING"]), + # Matrices label a native DSpark drafter either mtp or draft_model. + "SPEC_DECODING": (speculation, environment["SPEC_DECODING"].replace("draft_model", "mtp")), "AgentX client": (benchmark.get("command", "").endswith("srt_agentic.sh"), agentic), } if not agentic: From fab28a420e00cf5dd749c41619fbffaed274c9dc Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:28:54 -0500 Subject: [PATCH 040/105] feat(agentx): port the DSV4.1 Flash H100 and H200 vLLM AgentX configs to srt-slurm --- .../dsv41flash/vllm/h100-fp4-mtp/agentic.yaml | 78 ++++++++++++++++++ .../dsv41flash/vllm/h200-fp4-mtp/agentic.yaml | 82 +++++++++++++++++++ configs/nvidia-master.yaml | 6 +- 3 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..63d067ec79 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml @@ -0,0 +1,78 @@ +# DeepSeek-V4.1-Flash AgentX on H100 with vLLM native DSpark +# (https://github.com/vllm-project/recipes/blob/main/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml). +# At 1M context the sparse-attention indexer allocates a [max-num-batched-tokens, +# max-model-len] fp8 logits buffer during startup profiling (8192 x 1048576 x 2 B = +# 16 GiB), which OOMs next to ~36 GiB/GPU of weights on 80 GB cards. Capping +# batched tokens shrinks it; capping max-model-len would force the 256k-capped +# corpus onto a 1M-context model. Engram weights use UVA DRAM; the KV cache is +# GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-h100-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 + precision: fp4 + resources: + gpu_type: h100 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + engram-config: '{"cpu_offload":true}' + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + # The indexer buffer scales linearly with batched tokens: 4096 puts it at + # 8 GiB, and anything larger did not fit next to the weights on this SKU. + max-num-batched-tokens: 4096 + gpu-memory-utilization: 0.92 + disable-uvicorn-access-log: true + env: + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + # The indexer buffer is large enough that allocator fragmentation costs a KV block. + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + PYTHONUNBUFFERED: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. The scheduler batch is 2x CONC (AgentX subagent +# fan-out) instead of vLLM's 1024 default; graph capture is the smallest power of +# two covering that batch x (1 + 5 drafts), up to 2048. +override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 2, max-cudagraph-capture-size: 16}}}, benchmark: {env: {CONC: '1'}}} +override_tp8_c2: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 4, max-cudagraph-capture-size: 32}}}, benchmark: {env: {CONC: '2'}}} +override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 8, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '4'}}} +override_tp8_c8: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 16, max-cudagraph-capture-size: 128}}}, benchmark: {env: {CONC: '8'}}} +override_tp8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 32, max-cudagraph-capture-size: 256}}}, benchmark: {env: {CONC: '16'}}} +override_tp8_c20: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 40, max-cudagraph-capture-size: 256}}}, benchmark: {env: {CONC: '20'}}} +override_tp8_c24: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 48, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '24'}}} +override_tp8_c28: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 56, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '28'}}} diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..cfbbeb835e --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml @@ -0,0 +1,82 @@ +# DeepSeek-V4.1-Flash AgentX on H200 with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). Engram weights use +# UVA DRAM; the KV cache is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-h200-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 + precision: fp4 + resources: + gpu_type: h200 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 720 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + engram-config: '{"cpu_offload":true}' + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + # The HF cache here is a VIRTIOFS mount, which vLLM does not treat as a + # network FS, so it memory-maps the 475 GiB checkpoint lazily: 19/48 shards + # loaded in the 3600 s readiness window (run 35012494184). Stream the shards + # into page cache with parallel readers first. + safetensors-load-strategy: prefetch + disable-uvicorn-access-log: true + env: + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Graph capture starts at 64 tokens (DSpark5 verification +# for concurrent subagents at c1-c4) and doubles until it covers CONC x (1 + 5 +# drafts), up to 2048. TP4 leaves ~15 GiB of KV per 141 GB GPU, so it caps +# batched tokens at 4096 (the indexer's 1M-wide logits buffer), bounds the +# scheduler at 2x CONC within [16, 256] (FlashInfer autotune fails below 16) and +# stops capturing above 512 tokens. +override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '1'}}} +override_tp8_c2: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '2'}}} +override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '4'}}} +override_tp8_c8: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '8'}}} +override_tp8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 128}}}, benchmark: {env: {CONC: '16'}}} +override_tp8_c32: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 256}}}, benchmark: {env: {CONC: '32'}}} +override_tp8_c64: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '64'}}} +override_tp8_c128: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 1024}}}, benchmark: {env: {CONC: '128'}}} +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '1'}}} +override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '2'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '4'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 128, max-num-batched-tokens: 4096, max-num-seqs: 32}}}, benchmark: {env: {CONC: '16'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 256, max-num-batched-tokens: 4096, max-num-seqs: 64}}}, benchmark: {env: {CONC: '32'}}} +override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 128}}}, benchmark: {env: {CONC: '64'}}} +override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 256}}}, benchmark: {env: {CONC: '128'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6e42526a27..f04af20385 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8326,7 +8326,7 @@ dsv41flash-fp4-h100-vllm-agentic-dspark: # Extend beyond the c1 full-context capacity estimate to measure the # AgentX workload's practical saturation curve; c20, c24 and c28 fill in # the knee between c16 and the point where 1M-token trajectories preempt. - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 20, 24, 28] } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 20, 24, 28], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml } # SGLang arm for DeepSeek-V4.1-Flash AgentX on H100, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). @@ -8515,13 +8515,13 @@ dsv41flash-fp4-h200-vllm-agentic-dspark: search-space: # 8x141 GB holds the 511 GB checkpoint minus the UVA-offloaded Engram # tables, so the KV cache stays GPU-resident across the full range. - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml } # TP4 doubles the per-GPU share to ~99 GiB of weights on each 141 GB H200 # with the Engram tables offloaded, leaving ~15 GiB of KV per GPU (~17M # tokens across the four ranks, about 3x the pool that served c1-c128 on # B200 TP2). The script caps batched tokens and the scheduler batch on # this arm so the sparse-attention indexer buffer does not consume it. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml } # SGLang arm for DeepSeek-V4.1-Flash AgentX on H200, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). From f9f470b29a496946b72296865dda9446bab4ab6a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:28:54 -0500 Subject: [PATCH 041/105] feat(agentx): port the MiniMax-M3 H100 and H200 vLLM AgentX configs to srt-slurm --- .../configs/vllm-mooncake-0.3.11.sh | 9 ++ .../minimaxm3/vllm/h100-fp8-mtp/agentic.yaml | 145 ++++++++++++++++++ .../minimaxm3/vllm/h200-fp8-mtp/agentic.yaml | 112 ++++++++++++++ configs/nvidia-master.yaml | 8 +- 4 files changed, 270 insertions(+), 4 deletions(-) create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/vllm-mooncake-0.3.11.sh create mode 100644 benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml create mode 100644 benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-mooncake-0.3.11.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-mooncake-0.3.11.sh new file mode 100755 index 0000000000..74b8c63c3c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-mooncake-0.3.11.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Pin the worker's Mooncake store client to the release the master runs. +set -euo pipefail +pip_install=(python3 -m pip install) +if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages; then + pip_install+=(--break-system-packages) +fi +"${pip_install[@]}" --quiet --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 +python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..fb6b0f6475 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml @@ -0,0 +1,145 @@ +# MiniMax-M3 MXFP8 AgentX on H100 with vLLM EAGLE3 (the GQA draft head) and +# optional Mooncake DRAM KV offload. 26 GiB of weights per GPU live in host memory. +base: + schema: 2 + name: minimaxm3-fp8-h100-vllm-agentic + model: + path: hf:MiniMaxAI/MiniMax-M3-MXFP8 + container: vllm/vllm-openai:v0.27.1 + precision: fp8 + resources: + gpu_type: h100 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: MiniMaxAI/MiniMax-M3-MXFP8 + tensor-parallel-size: 8 + data-parallel-size: 1 + gpu-memory-utilization: 0.90 + cpu-offload-gb: 26 + attention-backend: TRITON_ATTN + safetensors-load-strategy: lazy + kv-cache-dtype: fp8 + block-size: 128 + language-model-only: true + enable-prefix-caching: true + enable-prompt-tokens-details: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + # Three-token EAGLE3 with the GQA draft head. Throughput runs switch to + # synthetic rejection at the golden acceptance length. + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tool-call-parser: minimax_m3 + reasoning-parser: minimax_m3 + enable-auto-tool-choice: true + trust-remote-code: true + env: + PYTHONNOUSERSITE: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: MiniMaxAI/MiniMax-M3-MXFP8 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + +# One variant per point. The scheduler batch is 2x CONC and graph capture covers +# it at four tokens per sequence (three drafts). +override_tp8_c1: {roles: {agg: {args: {max-num-seqs: 2, max-cudagraph-capture-size: 8}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp8_c2: {roles: {agg: {args: {max-num-seqs: 4, max-cudagraph-capture-size: 16}}}, benchmark: {env: {CONC: '2', KV_OFFLOADING: none}}} +override_tp8_c3: {roles: {agg: {args: {max-num-seqs: 6, max-cudagraph-capture-size: 24}}}, benchmark: {env: {CONC: '3', KV_OFFLOADING: none}}} +override_tp8_c4: {roles: {agg: {args: {max-num-seqs: 8, max-cudagraph-capture-size: 32}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp8_c5: {roles: {agg: {args: {max-num-seqs: 10, max-cudagraph-capture-size: 40}}}, benchmark: {env: {CONC: '5', KV_OFFLOADING: none}}} +# DRAM points offload KV to an embedded Mooncake store. Per rank: the host budget +# (1731 GB = 1612 GiB) less the 414 GiB checkpoint page cache, over TP8, less the +# 26 GiB CPU weight offload and the 4 GiB local buffer = 119 GB. +override_tp8_c6_dram: + # The worker's Mooncake client and the master run the same pinned release. + setup_script: vllm-mooncake-0.3.11.sh + services: + - name: mooncake-master + type: mooncake-master + preamble: >- + pip_install=(python3 -m pip install); + python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages && pip_install+=(--break-system-packages); + "${pip_install[@]}" --quiet --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 + args: ['--eviction_high_watermark_ratio=0.80', '--eviction_ratio=0.10'] + options: + store_config: + mode: embedded + metadata_server: P2PHANDSHAKE + global_segment_size: 119GB + local_buffer_size: 4GB + protocol: rdma + device_name: '' + enable_offload: false + roles: + agg: + args: + max-num-seqs: 12 + max-cudagraph-capture-size: 48 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' + env: + PYTHONHASHSEED: '0' + MC_SLICE_SIZE: '1048576' + MC_WORKERS_PER_CTX: '4' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + benchmark: + env: + CONC: '6' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' +override_tp8_c8_dram: + # The worker's Mooncake client and the master run the same pinned release. + setup_script: vllm-mooncake-0.3.11.sh + services: + - name: mooncake-master + type: mooncake-master + preamble: >- + pip_install=(python3 -m pip install); + python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages && pip_install+=(--break-system-packages); + "${pip_install[@]}" --quiet --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 + args: ['--eviction_high_watermark_ratio=0.80', '--eviction_ratio=0.10'] + options: + store_config: + mode: embedded + metadata_server: P2PHANDSHAKE + global_segment_size: 119GB + local_buffer_size: 4GB + protocol: rdma + device_name: '' + enable_offload: false + roles: + agg: + args: + max-num-seqs: 16 + max-cudagraph-capture-size: 64 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' + env: + PYTHONHASHSEED: '0' + MC_SLICE_SIZE: '1048576' + MC_WORKERS_PER_CTX: '4' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + benchmark: + env: + CONC: '8' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..c0f449c1e7 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml @@ -0,0 +1,112 @@ +# MiniMax-M3 MXFP8 AgentX on H200 with vLLM EAGLE3 (the GQA draft head) and +# optional Mooncake DRAM KV offload. +base: + schema: 2 + name: minimaxm3-fp8-h200-vllm-agentic + model: + path: hf:MiniMaxAI/MiniMax-M3-MXFP8 + container: vllm/vllm-openai:nightly-d9105ea8001e0a6d77a96327d17515bb5791fb36 + precision: fp8 + resources: + gpu_type: h200 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: MiniMaxAI/MiniMax-M3-MXFP8 + tensor-parallel-size: 8 + data-parallel-size: 1 + attention-backend: FLASH_ATTN + # Humming for dense MXFP8 linear layers. + linear-backend: humming + kv-cache-dtype: fp8 + block-size: 128 + language-model-only: true + enable-prefix-caching: true + enable-prompt-tokens-details: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + # Three-token EAGLE3 with the GQA draft head. Throughput runs switch to + # synthetic rejection at the golden acceptance length. + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tool-call-parser: minimax_m3 + reasoning-parser: minimax_m3 + enable-auto-tool-choice: true + trust-remote-code: true + env: + PYTHONNOUSERSITE: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + # Atomic reduction for the Marlin MXFP8 MoE path. + VLLM_MARLIN_USE_ATOMIC_ADD: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: MiniMaxAI/MiniMax-M3-MXFP8 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + +# One variant per point. The scheduler batch is 2x CONC and graph capture covers +# it at four tokens per sequence (three drafts). +override_tp8_c1: {roles: {agg: {args: {max-num-seqs: 2, max-cudagraph-capture-size: 8, gpu-memory-utilization: 0.95}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp8_c2: {roles: {agg: {args: {max-num-seqs: 4, max-cudagraph-capture-size: 16, gpu-memory-utilization: 0.95}}}, benchmark: {env: {CONC: '2', KV_OFFLOADING: none}}} +override_tp8_c4: {roles: {agg: {args: {max-num-seqs: 8, max-cudagraph-capture-size: 32, gpu-memory-utilization: 0.95}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp8_c6: {roles: {agg: {args: {max-num-seqs: 12, max-cudagraph-capture-size: 48, gpu-memory-utilization: 0.95}}}, benchmark: {env: {CONC: '6', KV_OFFLOADING: none}}} +override_tp8_c8: {roles: {agg: {args: {max-num-seqs: 16, max-cudagraph-capture-size: 64, gpu-memory-utilization: 0.95}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} +override_tp8_c10: {roles: {agg: {args: {max-num-seqs: 20, max-cudagraph-capture-size: 80, gpu-memory-utilization: 0.95}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: none}}} +# The DRAM point offloads KV to an embedded Mooncake store, leaving HBM headroom +# at 0.90. Per rank: the host budget (1234 GB = 1149 GiB) less the 414 GiB +# checkpoint page cache, over TP8, less the 4 GiB local buffer = 87 GB. +override_tp8_c12_dram: + # The worker's Mooncake client and the master run the same pinned release. + setup_script: vllm-mooncake-0.3.11.sh + services: + - name: mooncake-master + type: mooncake-master + preamble: >- + pip_install=(python3 -m pip install); + python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages && pip_install+=(--break-system-packages); + "${pip_install[@]}" --quiet --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 + args: ['--eviction_high_watermark_ratio=0.80', '--eviction_ratio=0.10'] + options: + store_config: + mode: embedded + metadata_server: P2PHANDSHAKE + global_segment_size: 87GB + local_buffer_size: 4GB + protocol: rdma + device_name: '' + enable_offload: false + roles: + agg: + args: + gpu-memory-utilization: 0.9 + max-num-seqs: 24 + max-cudagraph-capture-size: 96 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' + env: + PYTHONHASHSEED: '0' + MC_SLICE_SIZE: '1048576' + MC_WORKERS_PER_CTX: '4' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + benchmark: + env: + CONC: '12' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1234' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f04af20385..8ef80883de 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5421,8 +5421,8 @@ minimaxm3-fp8-h100-vllm-agentic-mtp: # The fast sweep places the resident HBM cliff between c5 and c6. - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [6, 8] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [6, 8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml } minimaxm3-fp8-h200-vllm-agentic-mtp: image: vllm/vllm-openai:nightly-d9105ea8001e0a6d77a96327d17515bb5791fb36 @@ -5437,8 +5437,8 @@ minimaxm3-fp8-h200-vllm-agentic-mtp: # Keep the resident c1-c10 points and the Mooncake DRAM-offload c12 point. - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10] } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml } qwen3.5-fp4-b200-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 From 768e383a238224199ad9f3fb4f51a05fa630bd1e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:29:33 -0500 Subject: [PATCH 042/105] feat(agentx): port the Qwen3.5 MI355X SGLang AgentX config to srt-slurm --- .../sglang/mi355x-fp4-mtp/agentic.yaml | 85 +++++++++++++++++++ configs/amd-master.yaml | 6 +- 2 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..7fc43af170 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,85 @@ +# Qwen3.5-397B-A17B MXFP4 AgentX on MI355X with SGLang native EAGLE MTP; KV is +# GPU-resident or extends to host DRAM through HiCache. +base: + schema: 2 + name: qwen3.5-fp4-mi355x-sglang-agentic + model: + path: hf:amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 + container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 + tokenizer-path: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 + trust-remote-code: true + data-parallel-size: 1 + expert-parallel-size: 1 + attention-backend: aiter + mem-fraction-static: 0.8 + model-loader-extra-config: '{"enable_multithread_load": true}' + watchdog-timeout: 1200 + page-size: 16 + kv-cache-dtype: fp8_e4m3 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + scheduler-recv-interval: 60 + stream-interval: 50 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-metrics: true + enable-cache-report: true + env: + PYTHONNOUSERSITE: '1' + SGLANG_USE_AITER: '1' + SGLANG_USE_AITER_UNIFIED_ATTN: '1' + AITER_FLYDSL_FORCE: '1' + SGLANG_MAMBA_SSM_DTYPE: bfloat16 + ROCM_QUICK_REDUCE_QUANTIZATION: INT4 + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 + # 256k-capped corpus within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. Admission is 2x CONC and the decode graph batch follows +# it up to 128; TP4 adds tokenizer workers. HiCache holds 1.5x the device KV pool. +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 2, cuda-graph-max-bs-decode: 2}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 8, cuda-graph-max-bs-decode: 8}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 16, cuda-graph-max-bs-decode: 16}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} +override_tp4_c12: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 24, cuda-graph-max-bs-decode: 24}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: none}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 32, cuda-graph-max-bs-decode: 32}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: none}}} +override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 2, cuda-graph-max-bs-decode: 2}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 8, cuda-graph-max-bs-decode: 8}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 16, cuda-graph-max-bs-decode: 16}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} +override_tp2_c12: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 24, cuda-graph-max-bs-decode: 24}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: none}}} +override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 32, cuda-graph-max-bs-decode: 32}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: none}}} +override_tp2_c20: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 40, cuda-graph-max-bs-decode: 40}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: none}}} +override_tp2_c20_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 40, cuda-graph-max-bs-decode: 40, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: kernel, hicache-mem-layout: page_first}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: dram}}} +override_tp2_c24_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 48, cuda-graph-max-bs-decode: 48, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: kernel, hicache-mem-layout: page_first}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: dram}}} +override_tp2_c28_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 56, cuda-graph-max-bs-decode: 56, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: kernel, hicache-mem-layout: page_first}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: dram}}} +override_tp2_c32_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 64, cuda-graph-max-bs-decode: 64, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: kernel, hicache-mem-layout: page_first}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: dram}}} +override_tp2_c36_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 72, cuda-graph-max-bs-decode: 72, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: kernel, hicache-mem-layout: page_first}}}, benchmark: {env: {CONC: '36', KV_OFFLOADING: dram}}} +override_tp2_c40_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 80, cuda-graph-max-bs-decode: 80, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: kernel, hicache-mem-layout: page_first}}}, benchmark: {env: {CONC: '40', KV_OFFLOADING: dram}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 89cd673dac..7fe6809c36 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -420,9 +420,9 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 36, 40] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 36, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 From dddcf1fa32f86cae100a25d0a59bc7fa8f51f07d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:29:33 -0500 Subject: [PATCH 043/105] feat(agentx): port the DSV4.1 Flash MI355X SGLang AgentX config to srt-slurm --- .../sglang/mi355x-fp4-mtp/agentic.yaml | 90 +++++++++++++++++++ configs/amd-master.yaml | 2 +- 2 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..0a1fdec554 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,90 @@ +# DeepSeek-V4.1-Flash AgentX on MI355X with SGLang native DSpark, following the +# cookbook's MI350X TP4/EP4 cell with radix caching enabled for AgentX prefix +# reuse. The KV cache and TP4 Engram tables are GPU-resident. +# https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 +base: + schema: 2 + name: dsv41flash-fp4-mi355x-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: lmsysorg/sglang:dev-dsv41-mi35x@sha256:c609f74f01b206af9664fb66da8ebef47642a7328a2478eabeebb352e014661a + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 4 + # C32 still exhausted HBM at 0.80 with 4096-token chunks; reserve + # another 10% of physical HBM for native long-prefill scratch. + mem-fraction-static: 0.7 + # Native FP4 prefill scratch scales with query tokens times context. + chunked-prefill-size: 4096 + # The compressed-KV store computes byte offsets in signed int32. Keep + # ratio-1 pages below 2 GiB without reducing the 1M context limit. + max-total-tokens: 3145728 + # C32 fanout reached 63 running requests before a native HIP illegal + # access. Bound admission; additional client requests remain queued. + max-running-requests: 32 + # DSpark is the checkpoint's bundled draft; the block size is its only knob. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + cuda-graph-max-bs-decode: 64 + # Breakable prefill hit HIP illegal access; prefill runs eagerly. + cuda-graph-backend-prefill: disabled + reasoning-parser: auto + tool-call-parser: auto + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV41_REASONING_EFFORT: high + # The preview image's native kernels, allocator and tuning CSV; TP4 + # Engram stays GPU-resident as in the official MI350X recipe. + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + SGLANG_USE_AITER: '1' + SGLANG_MOE_PADDING: '1' + AITER_FLYDSL_FORCE_REDUCE: '1' + ROCM_QUICK_REDUCE_QUANTIZATION: NONE + # The allocator only garbage-collects below a 1.0 memory fraction; + # reclaim unused long-prefill blocks for HIP/RCCL at 80% of 99%. + PYTORCH_HIP_ALLOC_CONF: garbage_collection_threshold:0.8,per_process_memory_fraction:0.99 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Full-context corpus for this 1M-context recipe. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point; the saturation point gets a longer warmup drain. +override_tp4_c1: {benchmark: {env: {CONC: '1'}}} +override_tp4_c2: {benchmark: {env: {CONC: '2'}}} +override_tp4_c4: {benchmark: {env: {CONC: '4'}}} +override_tp4_c8: {benchmark: {env: {CONC: '8'}}} +override_tp4_c16: {benchmark: {env: {CONC: '16'}}} +override_tp4_c32: {benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 7fe6809c36..8f965b1631 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1424,4 +1424,4 @@ dsv41flash-fp4-mi355x-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.60 search-space: - - { tp: 4, ep: 4, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 4, ep: 4, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml } From 775e639ab4dedda7a312f14b80c9f33d484949d0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:29:33 -0500 Subject: [PATCH 044/105] feat(agentx): port the GLM-5.2 FP4 MI355X SGLang AgentX config to srt-slurm --- .../glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml | 69 +++++++++++++++++++ configs/amd-master.yaml | 4 +- 2 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..6e7956ce60 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,69 @@ +# GLM-5.2 FP4 AgentX on MI355X with SGLang EAGLE MTP. +base: + schema: 2 + name: glm5.2-fp4-mi355x-sglang-agentic + model: + path: hf:amd/GLM-5.2-MXFP4 + container: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260923 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: amd/GLM-5.2-MXFP4 + trust-remote-code: true + kv-cache-dtype: fp8_e4m3 + dsa-prefill-backend: triton + dsa-decode-backend: triton + # GLM-5.2 emits GLM-4.7-style tool calls; glm45 keeps hybrid thinking + # in reasoning_content. + tool-call-parser: glm47 + reasoning-parser: glm45 + # 32k chunks interleave decode between prefill chunks and keep per-chunk + # activation headroom low enough for 0.85. + chunked-prefill-size: 32768 + mem-fraction-static: 0.85 + speculative-algorithm: EAGLE + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + watchdog-timeout: 1800 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # The DSA indexer uses the precompiled HIP Top-K op. + SGLANG_OPT_USE_TOPK_V2: 'false' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: amd/GLM-5.2-MXFP4 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC; the decode graph batch follows it +# up to 64. The TP4 HiCache tier is 180 GB per rank. +override_tp4_c4_hicache: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, max-running-requests: 8, cuda-graph-max-bs-decode: 8, enable-hierarchical-cache: true, hicache-size: 180, hicache-write-policy: write_through}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: dram}}} +override_tp4_c8_hicache: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, max-running-requests: 16, cuda-graph-max-bs-decode: 16, enable-hierarchical-cache: true, hicache-size: 180, hicache-write-policy: write_through}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: dram}}} +override_tp4_c10_hicache: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, max-running-requests: 20, cuda-graph-max-bs-decode: 20, enable-hierarchical-cache: true, hicache-size: 180, hicache-write-policy: write_through}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: dram}}} +override_tp4_c12_hicache: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, max-running-requests: 24, cuda-graph-max-bs-decode: 24, enable-hierarchical-cache: true, hicache-size: 180, hicache-write-policy: write_through}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: dram}}} +override_tp4_c14_hicache: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, max-running-requests: 28, cuda-graph-max-bs-decode: 28, enable-hierarchical-cache: true, hicache-size: 180, hicache-write-policy: write_through}}}, benchmark: {env: {CONC: '14', KV_OFFLOADING: dram}}} +override_tp4_c16_hicache: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, max-running-requests: 32, cuda-graph-max-bs-decode: 32, enable-hierarchical-cache: true, hicache-size: 180, hicache-write-policy: write_through}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: dram}}} +override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 2, cuda-graph-max-bs-decode: 2}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp8_c2: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 4, cuda-graph-max-bs-decode: 4}}}, benchmark: {env: {CONC: '2', KV_OFFLOADING: none}}} +override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 8, cuda-graph-max-bs-decode: 8}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 8f965b1631..b93f82d662 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1258,8 +1258,8 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.85 search-space: - - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 10, 12, 14, 16], spec-decoding: mtp } - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp } + - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 10, 12, 14, 16], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml } # GLM-5.2 FP8 agentic-coding benchmark on MI355X via SGLang with MTP speculative # decoding: the FP8 precision sibling of glm5.2-fp4-mi355x-sglang-agentic-mtp, From 84d67d1ad8ecfb09a12ce55667b0128aa76428c3 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:29:33 -0500 Subject: [PATCH 045/105] feat(agentx): port the GLM-5.2 FP8 MI355X SGLang AgentX config to srt-slurm --- .../glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml | 68 +++++++++++++++++++ configs/amd-master.yaml | 4 +- 2 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..0fc956f57d --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml @@ -0,0 +1,68 @@ +# GLM-5.2 FP8 AgentX on MI355X with SGLang EAGLE MTP. The ~756 GB +# block-FP8 checkpoint is TP8-only; quantization is detected from the checkpoint. +base: + schema: 2 + name: glm5.2-fp8-mi355x-sglang-agentic + model: + path: hf:zai-org/GLM-5.2-FP8 + container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260917 + precision: fp8 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + kv-cache-dtype: fp8_e4m3 + dsa-prefill-backend: tilelang + dsa-decode-backend: tilelang + # GLM-5.2 emits GLM-4.7-style tool calls; glm45 keeps hybrid thinking + # in reasoning_content. + tool-call-parser: glm47 + reasoning-parser: glm45 + # 32k chunks interleave decode between prefill chunks and keep per-chunk + # activation headroom low enough for 0.85. + chunked-prefill-size: 32768 + mem-fraction-static: 0.85 + speculative-algorithm: EAGLE + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + watchdog-timeout: 1800 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # The DSA indexer uses the precompiled HIP Top-K op. + SGLANG_OPT_USE_TOPK_V2: 'false' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: zai-org/GLM-5.2-FP8 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC; the decode graph batch follows it +# up to 64. HiCache holds 1.5x the device KV pool and skips non-reusable blocks. +override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 2, cuda-graph-max-bs-decode: 2}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp8_c2: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 4, cuda-graph-max-bs-decode: 4}}}, benchmark: {env: {CONC: '2', KV_OFFLOADING: none}}} +override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 8, cuda-graph-max-bs-decode: 8}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp8_c10: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 20, cuda-graph-max-bs-decode: 20}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: none}}} +override_tp8_c8_hicache: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 16, cuda-graph-max-bs-decode: 16, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through_selective, hicache-io-backend: direct, hicache-mem-layout: page_first_direct}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: dram}}} +override_tp8_c12_hicache: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 24, cuda-graph-max-bs-decode: 24, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through_selective, hicache-io-backend: direct, hicache-mem-layout: page_first_direct}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: dram}}} +override_tp8_c16_hicache: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 32, cuda-graph-max-bs-decode: 32, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through_selective, hicache-io-backend: direct, hicache-mem-layout: page_first_direct}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: dram}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b93f82d662..bf2368810e 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1288,8 +1288,8 @@ glm5.2-fp8-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 10], spec-decoding: mtp } - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8, 12, 16], spec-decoding: mtp } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 10], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8, 12, 16], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM with MTP speculative # decoding, matching the standalone recipe (ROCm/ATOM PR 2345). Small # concurrency (C2-C10) is TP4-only on the GPU prefix cache with no KV offload; From 4625b67df6edc8317dbbde361a4f31de06ce1e8f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:29:33 -0500 Subject: [PATCH 046/105] feat(agentx): port the DSV4 MI355X SGLang AgentX config to srt-slurm --- .../dsv4/sglang/mi355x-fp4-mtp/agentic.yaml | 94 +++++++++++++++++++ configs/amd-master.yaml | 6 +- 2 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..8405cf812c --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,94 @@ +# DeepSeek-V4-Pro-0813 AgentX on MI355X with SGLang DSpark. Low concurrency +# keeps KV on GPU; higher concurrency extends it to host DRAM through HiCache, +# and the widest band runs DP attention behind a cache-aware SGLang router. +base: + schema: 2 + name: dsv4-fp4-mi355x-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 + container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260914 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + attention-backend: dsv4 + enable-deepseek-v4-fp4-indexer: true + page-size: 256 + swa-full-tokens-ratio: 0.1 + kv-cache-dtype: fp8_e4m3 + enforce-shared-experts-fusion: true + # No chat-template override: deepseek_v4_thinking.jinja drops tool + # definitions and tool messages. + tool-call-parser: deepseekv4 + reasoning-parser: deepseek-v4 + chunked-prefill-size: 16384 + mem-fraction-static: 0.86 + prefill-decode-interval: 20 + # DSpark is the checkpoint's bundled draft; gamma 6 is AL-optimal on the + # golden curve and verifies a window of 7. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_USE_ROCM700A: '0' + SGLANG_HACK_FLASHMLA_BACKEND: unified_kv_triton + AITER_BF16_FP8_MOE_BOUND: '0' + TORCH_BLAS_PREFER_HIPBLASLT: '1' + HSA_NO_SCRATCH_RECLAIM: '0' + # aiter batched GEMM for the absorbed MLA projections. + SGLANG_OPT_USE_AITER_BATCHED_GEMM: '1' + # Unified radix tree that releases out-of-window SWA slots, so cached + # sessions keep their trailing window under multi-turn load. + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: '1' + GPU_MAX_HW_QUEUES: '2' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC; the decode graph batch follows it +# up to 128. HiCache pins 1.5x the device KV pool per rank. DP attention keeps +# 8192 prefill tokens per rank, and the router pins each AIPerf session to the +# rank holding its prefix. Saturation points get a longer warmup drain. +override_tp8_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs-decode: 2}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp8_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs-decode: 8}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp8_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs-decode: 32}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: none}}} +override_tp8_c32_hicache: {roles: {agg: {args: {max-running-requests: 64, cuda-graph-max-bs-decode: 64, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: dram, AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp8_c48_hicache: {roles: {agg: {args: {max-running-requests: 96, cuda-graph-max-bs-decode: 96, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct}}}, benchmark: {env: {CONC: '48', KV_OFFLOADING: dram, AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp8_c128_dp_hicache: {frontend: {type: sglang-router, args: {policy: cache_aware, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {max-running-requests: 256, cuda-graph-max-bs-decode: 128, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, data-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-prefill-delayer: true, enable-dp-attention-local-control-broadcast: true, tokenizer-worker-num: 8, stream-interval: 20, prefill-delayer-token-usage-low-watermark: 0.7, chunked-prefill-size: 65536, mem-fraction-static: 0.92}, env: {SGLANG_SHARED_EXPERT_TP1: '1', SGLANG_DP_SHARED_EXPERT_LOCAL: '1', SGLANG_DP_USE_GATHERV: '1', SGLANG_DP_USE_REDUCE_SCATTER: '1', GPU_MAX_HW_QUEUES: '5'}}}, benchmark: {env: {CONC: '128', KV_OFFLOADING: dram, AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp8_c256_dp_hicache: {frontend: {type: sglang-router, args: {policy: cache_aware, balance-abs-threshold: 32, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {max-running-requests: 512, cuda-graph-max-bs-decode: 128, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, data-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-prefill-delayer: true, enable-dp-attention-local-control-broadcast: true, tokenizer-worker-num: 8, stream-interval: 20, prefill-delayer-token-usage-low-watermark: 0.7, chunked-prefill-size: 65536, mem-fraction-static: 0.92}, env: {SGLANG_SHARED_EXPERT_TP1: '1', SGLANG_DP_SHARED_EXPERT_LOCAL: '1', SGLANG_DP_USE_GATHERV: '1', SGLANG_DP_USE_REDUCE_SCATTER: '1', GPU_MAX_HW_QUEUES: '5'}}}, benchmark: {env: {CONC: '256', KV_OFFLOADING: dram, AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index bf2368810e..6afe812e6b 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1328,9 +1328,9 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 16], spec-decoding: draft_model } - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 48], spec-decoding: draft_model } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [128, 256], spec-decoding: draft_model } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 16], spec-decoding: draft_model, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 48], spec-decoding: draft_model, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [128, 256], spec-decoding: draft_model, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml } # Upstream ROCm nightly rather than the deepseekv41-flash-0909 release tag: that # tag predates vllm-project/vllm#56503, which moves the mHC delayed pre block off From 31825ceb2097dce5ded0b6c07a1ab62734864f1f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:29:33 -0500 Subject: [PATCH 047/105] feat(agentx): port the DSV4 MI355X ATOM AgentX config to srt-slurm --- .../dsv4/atom/mi355x-fp4-mtp/agentic.yaml | 73 +++++++++++++++++++ configs/amd-master.yaml | 4 +- 2 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..e7d4f74475 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,73 @@ +# DeepSeek-V4-Pro-0813 AgentX on MI355X with ATOM DSpark K6 (six draft tokens, +# a verify window of 7). Low concurrency is TP-only; the wide band runs DP +# attention with native RCCL expert parallelism. The KV cache is GPU-resident. +base: + schema: 2 + name: dsv4-fp4-mi355x-atom-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 + container: rocm/atom-dev:nightly_202609161445 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: atomesh + container_image: rocm/atom-dev@sha256:f00a9dc588a380038fdad54cd57a956043cb316ddda54ebb1b62a24f8caa343b + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: atom + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + timeout-keep-alive: 900 + data-parallel-size: 1 + index-cache-dtype: fp4 + enable-prefix-caching: true + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + attn-prefill-chunk-size: 16384 + state-checkpoint-interval-tokens: 8192 + level: 3 + cudagraph-mode: FULL + method: dspark + num-speculative-tokens: 6 + env: + AITER_BF16_FP8_MOE_BOUND: '0' + AITER_LOG_LEVEL: WARNING + ATOM_MOE_GU_ITLV: '1' + ATOM_DISABLE_MMAP: 'true' + ATOM_DEBUG_PREFIX_HITS: '1' + ATOM_PROFILER_MORE: '0' + ATOM_PROFILER_TIMEOUT: '1200' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + # Long AgentX stalls exceed AIPerf's 30 s TCP_USER_TIMEOUT default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' + +# One variant per point. Admission is 2x CONC. KV is BF16 through concurrency 16 +# and FP8 in the DEP band. DEP pins sessions to one DP-attention rank so later +# turns reach their prefix KV, and captures every small per-rank decode batch. +override_tp8_c1: {roles: {agg: {args: {max-num-seqs: 2, kv-cache-dtype: bf16}}}, benchmark: {env: {CONC: '1'}}} +override_tp8_c2: {roles: {agg: {args: {max-num-seqs: 4, kv-cache-dtype: bf16}}}, benchmark: {env: {CONC: '2'}}} +override_tp8_c4: {roles: {agg: {args: {max-num-seqs: 8, kv-cache-dtype: bf16}}}, benchmark: {env: {CONC: '4'}}} +override_tp8_c8: {roles: {agg: {args: {max-num-seqs: 16, kv-cache-dtype: bf16}}}, benchmark: {env: {CONC: '8'}}} +override_tp8_c16: {roles: {agg: {args: {max-num-seqs: 32, kv-cache-dtype: bf16}}}, benchmark: {env: {CONC: '16'}}} +override_tp8_c48_dep: {roles: {agg: {args: {max-num-seqs: 96, kv-cache-dtype: fp8, enable-expert-parallel: true, enable-dp-attention: true, all2all-backend: rccl, dp-load-balance: least_tokens, moe-backend: standard, cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]'}, env: {ATOM_DP_SESSION_AFFINITY: '1', ATOM_DP_LB_REQ_EQUIV: '512', ATOM_ENABLE_PREFILL_DELAYER: '1', ATOM_PREFILL_DECODE_INTERVAL: '10'}}}, benchmark: {env: {CONC: '48', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp8_c64_dep: {roles: {agg: {args: {max-num-seqs: 128, kv-cache-dtype: fp8, enable-expert-parallel: true, enable-dp-attention: true, all2all-backend: rccl, dp-load-balance: least_tokens, moe-backend: standard, cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]'}, env: {ATOM_DP_SESSION_AFFINITY: '1', ATOM_DP_LB_REQ_EQUIV: '512', ATOM_ENABLE_PREFILL_DELAYER: '1', ATOM_PREFILL_DECODE_INTERVAL: '10'}}}, benchmark: {env: {CONC: '64', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp8_c96_dep: {roles: {agg: {args: {max-num-seqs: 192, kv-cache-dtype: fp8, enable-expert-parallel: true, enable-dp-attention: true, all2all-backend: rccl, dp-load-balance: least_tokens, moe-backend: standard, cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]'}, env: {ATOM_DP_SESSION_AFFINITY: '1', ATOM_DP_LB_REQ_EQUIV: '512', ATOM_ENABLE_PREFILL_DELAYER: '1', ATOM_PREFILL_DECODE_INTERVAL: '10'}}}, benchmark: {env: {CONC: '96', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp8_c128_dep: {roles: {agg: {args: {max-num-seqs: 256, kv-cache-dtype: fp8, enable-expert-parallel: true, enable-dp-attention: true, all2all-backend: rccl, dp-load-balance: least_tokens, moe-backend: standard, cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]'}, env: {ATOM_DP_SESSION_AFFINITY: '1', ATOM_DP_LB_REQ_EQUIV: '512', ATOM_ENABLE_PREFILL_DELAYER: '1', ATOM_PREFILL_DECODE_INTERVAL: '10'}}}, benchmark: {env: {CONC: '128', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp8_c256_dep: {roles: {agg: {args: {max-num-seqs: 512, kv-cache-dtype: fp8, enable-expert-parallel: true, enable-dp-attention: true, all2all-backend: rccl, dp-load-balance: least_tokens, moe-backend: standard, cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]'}, env: {ATOM_DP_SESSION_AFFINITY: '1', ATOM_DP_LB_REQ_EQUIV: '512', ATOM_ENABLE_PREFILL_DELAYER: '1', ATOM_PREFILL_DECODE_INTERVAL: '10'}}}, benchmark: {env: {CONC: '256', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 6afe812e6b..e30b180b8a 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -927,8 +927,8 @@ dsv4-fp4-mi355x-atom-agentic-mtp: scenarios: agentic-coding: - search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [48, 64, 96, 128, 256] } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [48, 64, 96, 128, 256], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml } dsr1-fp4-mi355x-sglang-disagg-mtp: image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 From 9cfaf0a532b9487118ae21cd2cb49c2b30371c3a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:29:33 -0500 Subject: [PATCH 048/105] feat(agentx): port the GLM-5.2 MI355X ATOM AgentX config to srt-slurm The DCP4 LMCache entry stays on the legacy script: srtctl reserves ATOM's kv-transfer-config for disaggregated workers. --- .../glm5.2/atom/mi355x-fp4-mtp/agentic.yaml | 59 +++++++++++++++++++ configs/amd-master.yaml | 4 +- 2 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..33ce6a593c --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,59 @@ +# GLM-5.2 MXFP4 AgentX on MI355X with ATOM MTP, TP-only with GPU-resident KV. +# The DCP4 LMCache band stays on the legacy script: srtctl reserves ATOM's +# kv-transfer-config for disaggregated workers. +base: + schema: 2 + name: glm5.2-fp4-mi355x-atom-agentic + model: + path: hf:amd/GLM-5.2-MXFP4 + container: rocm/atom-dev:nightly_202609211553 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: atomesh + container_image: rocm/atom-dev@sha256:f00a9dc588a380038fdad54cd57a956043cb316ddda54ebb1b62a24f8caa343b + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: atom + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: amd/GLM-5.2-MXFP4 + gpu-memory-utilization: 0.95 + enable_prefix_caching: true + # model.layers.78 is the MTP head, shipped in BF16; excluding it keeps + # online ptpc_fp8 quantization off the draft while experts stay MXFP4. + online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","model.layers.[0-9].mlp.*expert*","model.layers.[1-6][0-9].mlp.*expert*","model.layers.7[0-7].mlp.*expert*","model.layers.78.*"]}' + max-num-batched-tokens: 16384 + kv_cache_dtype: fp8 + index_cache_dtype: fp4 + block-size: 64 + method: mtp + env: + PYTHONNOUSERSITE: '1' + AITER_QUICK_REDUCE_QUANTIZATION: INT4 + AITER_USE_FLYDSL_MOE_SORTING: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: amd/GLM-5.2-MXFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. Admission is 2x CONC with graphs captured up to it. The +# draft is five tokens through concurrency 8 and four above it. +override_tp4_c2: {roles: {agg: {gpus: 4, args: {max-num-seqs: 4, cudagraph-capture-sizes: '[1,2,4]', num-speculative-tokens: 5}}}, benchmark: {env: {CONC: '2'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {max-num-seqs: 8, cudagraph-capture-sizes: '[1,2,4,8]', num-speculative-tokens: 5}}}, benchmark: {env: {CONC: '4'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {max-num-seqs: 16, cudagraph-capture-sizes: '[1,2,4,8,12,16]', num-speculative-tokens: 5}}}, benchmark: {env: {CONC: '8'}}} +override_tp4_c10: {roles: {agg: {gpus: 4, args: {max-num-seqs: 20, cudagraph-capture-sizes: '[1,2,4,8,12,16,20]', num-speculative-tokens: 4}}}, benchmark: {env: {CONC: '10'}}} +override_tp8_c1: {roles: {agg: {gpus: 8, args: {max-num-seqs: 2, cudagraph-capture-sizes: '[1,2]', num-speculative-tokens: 5}}}, benchmark: {env: {CONC: '1'}}} +override_tp8_c2: {roles: {agg: {gpus: 8, args: {max-num-seqs: 4, cudagraph-capture-sizes: '[1,2,4]', num-speculative-tokens: 5}}}, benchmark: {env: {CONC: '2'}}} +override_tp8_c4: {roles: {agg: {gpus: 8, args: {max-num-seqs: 8, cudagraph-capture-sizes: '[1,2,4,8]', num-speculative-tokens: 5}}}, benchmark: {env: {CONC: '4'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index e30b180b8a..a26e444ede 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1312,9 +1312,9 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: # Only the DCP4 arm offloads; the other two are GPU-resident. - dram-utilization: 0.171 search-space: - - { tp: 4, kv-offloading: none, conc-list: [2, 4, 8, 10], spec-decoding: mtp } + - { tp: 4, kv-offloading: none, conc-list: [2, 4, 8, 10], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml } - { tp: 4, dcp-size: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [16, 24, 32, 40, 48], spec-decoding: mtp } - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp } + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml } dsv4-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260914 From d133cbce9e5a04ffc2c57b76c79baf9b767c09bc Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:29:33 -0500 Subject: [PATCH 049/105] feat(agentx): port the Kimi-K3 MI355X ATOM AgentX config to srt-slurm The DCP8 LMCache entries stay on the legacy script: srtctl reserves ATOM's kv-transfer-config for disaggregated workers. --- .../configs/kimik3-atom-dspark-draft.sh | 13 +++ .../kimik3/atom/mi355x-fp4-mtp/agentic.yaml | 89 +++++++++++++++++++ configs/amd-master.yaml | 2 +- 3 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-atom-dspark-draft.sh create mode 100644 benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-atom-dspark-draft.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-atom-dspark-draft.sh new file mode 100644 index 0000000000..7419f48b64 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-atom-dspark-draft.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Stage the Kimi-K3 DSpark draft before ATOM starts: with an uncached repo id +# every rank pulls the same 7 GB at once. Shared-cache downloads can hit +# transient stale handles, hence the retries. +set -euo pipefail +unset HTTP_PROXY HTTPS_PROXY http_proxy https_proxy +for attempt in 1 2 3 4 5; do + hf download Inferact/Kimi-K3-DSpark && exit 0 + echo "hf download attempt $attempt failed; retrying in 60s" >&2 + sleep 60 +done +echo "hf download of Inferact/Kimi-K3-DSpark failed after 5 attempts" >&2 +exit 1 diff --git a/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..db711b975e --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,89 @@ +# Kimi-K3 MXFP4 AgentX on MI355X with ATOM DSpark: the interactive band, TP8 +# with GPU-resident KV and the deepest published draft (seven tokens). The +# 1.56 TB checkpoint only fits at TP8. The DCP8 LMCache bands stay on the legacy +# script: srtctl reserves ATOM's kv-transfer-config for disaggregated workers. +base: + schema: 2 + name: kimik3-fp4-mi355x-atom-agentic + model: + path: hf:moonshotai/Kimi-K3 + container: rocm/atom-dev:ubuntu24.04_py3.12_pytorch_release_2.10.0_kimi_k3_agentic_0911 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: atomesh + container_image: rocm/atom-dev@sha256:f00a9dc588a380038fdad54cd57a956043cb316ddda54ebb1b62a24f8caa343b + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: atom + setup_script: kimik3-atom-dspark-draft.sh + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: moonshotai/Kimi-K3 + trust-remote-code: true + # Fails closed if the chat template stops rendering a tools payload. + tool-call-parser: kimi_k3 + decode-context-parallel-size: 1 + kv_cache_dtype: fp8 + block-size: 128 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + gpu-memory-utilization: 0.9 + enable_prefix_caching: true + # -1 keeps the prompt-end checkpoint anchor without the fixed-interval grid. + state-checkpoint-interval-tokens: -1 + level: 3 + cudagraph-mode: FULL + # Global ptpc_fp8 online quantization; the trailing entries keep the BF16 + # draft (unprefixed layers.* and context_proj) unquantized. + online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*self_attn.[qkv]_conv1d*","*block_sparse_moe.experts*","*block_sparse_moe.routed_expert_*","*vision_tower*","*mm_projector*","layers.*.self_attn.fused_qkv_a_proj","layers.*.self_attn.q_b_proj","layers.*.self_attn.kv_b_proj","layers.*.self_attn.o_proj","layers.*.mlp.gate_up_proj","layers.*.mlp.down_proj","context_proj"]}' + method: dspark + draft-model: Inferact/Kimi-K3-DSpark + num-speculative-tokens: 7 + env: + # The image bakes in a build-host proxy that nothing serves on a node. + HTTP_PROXY: '' + HTTPS_PROXY: '' + http_proxy: '' + https_proxy: '' + NO_PROXY: localhost,127.0.0.1,::1 + no_proxy: localhost,127.0.0.1,::1 + PYTHONNOUSERSITE: '1' + ATOM_ENABLE_REPLAYSSM: '0' + AITER_REUSE_IDENTICAL_COMM_GROUPS: '0' + AITER_LOG_LEVEL: WARNING + AITER_SITUV2_A4W4: '1' + AITER_QUICK_REDUCE_QUANTIZATION: INT4 + AITER_FLYDSL_STAGE2_FP8: '1' + # Anchor-only state checkpointing. + ATOM_STATE_CHECKPOINT_DEMAND: '0' + ATOM_GDN_SSM_DTYPE: fp16 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: moonshotai/Kimi-K3 + HTTP_PROXY: '' + HTTPS_PROXY: '' + http_proxy: '' + https_proxy: '' + NO_PROXY: localhost,127.0.0.1,::1 + no_proxy: localhost,127.0.0.1,::1 + # Long agentic turns against a 1M context are prefill-bound on the server. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. Full graphs cover every verify batch up to 2x CONC +# times the eight-token verify window. +override_tp8_c1: {roles: {agg: {args: {cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]'}}}, benchmark: {env: {CONC: '1'}}} +override_tp8_c4: {roles: {agg: {args: {cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]'}}}, benchmark: {env: {CONC: '4'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index a26e444ede..40e3515364 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -701,7 +701,7 @@ kimik3-fp4-mi355x-atom-agentic-mtp: agentic-coding: - dram-utilization: 0.343 search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 4], spec-decoding: mtp } + - { tp: 8, kv-offloading: none, conc-list: [1, 4], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml } - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [14, 16, 48], spec-decoding: mtp } - dram-utilization: 0.513 search-space: From 269e704948360eae1fdb7bac8de27b3a4099c0eb Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:29:33 -0500 Subject: [PATCH 050/105] feat(agentx): port the MiniMax-M3 MI355X ATOM AgentX config to srt-slurm The LMCache entries stay on the legacy script: srtctl reserves ATOM's kv-transfer-config for disaggregated workers. --- .../configs/minimaxm3-atom-eagle3-deps.sh | 9 +++ .../atom/mi355x-fp4-mtp/agentic.yaml | 73 +++++++++++++++++++ configs/amd-master.yaml | 4 +- 3 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-atom-eagle3-deps.sh create mode 100644 benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-atom-eagle3-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-atom-eagle3-deps.sh new file mode 100644 index 0000000000..eb8f9016aa --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-atom-eagle3-deps.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Stage the MiniMax-M3 EAGLE3 GQA draft and the tokenizer dependencies the ATOM +# image lacks, in a side directory that the server's interpreter imports. +set -euo pipefail +hf download Inferact/MiniMax-M3-EAGLE3-GQA +deps=/tmp/inferencex-atom-runtime-deps +/opt/venv/bin/python -m pip install --quiet --target "$deps" --no-deps sentencepiece tiktoken +site=$(/opt/venv/bin/python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') +echo "$deps" > "$site/inferencex-atom-runtime-deps.pth" diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..62951121c5 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,73 @@ +# MiniMax-M3 MXFP4 AgentX on MI355X with ATOM EAGLE3 (GQA draft, three tokens) +# and GPU-resident KV at TP2 and TP4. The LMCache bands stay on the legacy +# script: srtctl reserves ATOM's kv-transfer-config for disaggregated workers. +base: + schema: 2 + name: minimaxm3-fp4-mi355x-atom-agentic + model: + path: hf:amd/MiniMax-M3-MXFP4 + container: rocm/atom-dev:nightly_202609171455 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: atomesh + container_image: rocm/atom-dev@sha256:f00a9dc588a380038fdad54cd57a956043cb316ddda54ebb1b62a24f8caa343b + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: atom + setup_script: minimaxm3-atom-eagle3-deps.sh + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: amd/MiniMax-M3-MXFP4 + trust-remote-code: true + kv_cache_dtype: fp8 + block-size: 128 + max-num-batched-tokens: 32768 + gpu-memory-utilization: 0.95 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,22,24,26,28,30,32,34,36,40,48,56,64]' + index-cache-dtype: fp8 + online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}' + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + method: eagle3 + draft-model: Inferact/MiniMax-M3-EAGLE3-GQA + num-speculative-tokens: 3 + env: + PYTHONNOUSERSITE: '1' + AITER_LOG_LEVEL: WARNING + AITER_SITUV2_A4W4: '1' + AITER_QUICK_REDUCE_QUANTIZATION: INT4 + AITER_FLYDSL_STAGE2_FP8: '1' + ATOM_FORCE_ATTN_TRITON: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: amd/MiniMax-M3-MXFP4 + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. Admission is 2x CONC. TP4 shards the indexer across +# ranks at the measured points (15, 20, 24, 28, 32). +override_tp4_c1: {roles: {agg: {gpus: 4, args: {max-num-seqs: 2}}}, benchmark: {env: {CONC: '1'}}} +override_tp4_c2: {roles: {agg: {gpus: 4, args: {max-num-seqs: 4}}}, benchmark: {env: {CONC: '2'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {max-num-seqs: 8}}}, benchmark: {env: {CONC: '4'}}} +override_tp4_c5: {roles: {agg: {gpus: 4, args: {max-num-seqs: 10}}}, benchmark: {env: {CONC: '5'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} +override_tp4_c10: {roles: {agg: {gpus: 4, args: {max-num-seqs: 20}}}, benchmark: {env: {CONC: '10'}}} +override_tp4_c12: {roles: {agg: {gpus: 4, args: {max-num-seqs: 24}}}, benchmark: {env: {CONC: '12'}}} +override_tp4_c15: {roles: {agg: {gpus: 4, args: {max-num-seqs: 30}, env: {ATOM_M3_INDEXER_CP: '1'}}}, benchmark: {env: {CONC: '15'}}} +override_tp4_c20: {roles: {agg: {gpus: 4, args: {max-num-seqs: 40}, env: {ATOM_M3_INDEXER_CP: '1'}}}, benchmark: {env: {CONC: '20'}}} +override_tp4_c24: {roles: {agg: {gpus: 4, args: {max-num-seqs: 48}, env: {ATOM_M3_INDEXER_CP: '1'}}}, benchmark: {env: {CONC: '24'}}} +override_tp4_c28: {roles: {agg: {gpus: 4, args: {max-num-seqs: 56}, env: {ATOM_M3_INDEXER_CP: '1'}}}, benchmark: {env: {CONC: '28'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {max-num-seqs: 64}, env: {ATOM_M3_INDEXER_CP: '1'}}}, benchmark: {env: {CONC: '32'}}} +override_tp2_c1: {roles: {agg: {gpus: 2, args: {max-num-seqs: 2}}}, benchmark: {env: {CONC: '1'}}} +override_tp2_c2: {roles: {agg: {gpus: 2, args: {max-num-seqs: 4}}}, benchmark: {env: {CONC: '2'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 40e3515364..7c2a344923 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -721,8 +721,8 @@ minimaxm3-fp4-mi355x-atom-agentic-mtp: # = TOTAL_CPU_DRAM_GB / TP = node_DRAM * dram-utilization / 8 (TP-independent). - dram-utilization: 0.687 search-space: - - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 5, 8, 10, 12, 15, 20, 24, 28, 32], spec-decoding: mtp } - - { tp: 2, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp } + - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 5, 8, 10, 12, 15, 20, 24, 28, 32], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml } - { tp: 2, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5rc3+rocm7.2.4" }, conc-list: [20, 25, 30], spec-decoding: mtp } - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5rc3+rocm7.2.4" }, conc-list: [40, 48], spec-decoding: mtp } From 4d56892e2e4418f0a1019d536756da4bb4deddbc Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:29:40 -0500 Subject: [PATCH 051/105] feat(srt): accept EAGLE3 speculation in the single-node adapter --- infx/srt_slurm/single_node.py | 5 +++-- utils/test_srt_single_node.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/infx/srt_slurm/single_node.py b/infx/srt_slurm/single_node.py index c9af2805c6..57983a1934 100644 --- a/infx/srt_slurm/single_node.py +++ b/infx/srt_slurm/single_node.py @@ -88,8 +88,9 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N if environment["FRAMEWORK"] not in {"sglang", "trt", "atom", "vllm"}: raise ValueError(f"Unsupported single-node framework: {environment['FRAMEWORK']!r}") spec = spec_parameters(role, engine) - if spec and spec["method"] not in {"eagle", "nextn", "mtp", "dspark"}: - raise ValueError("Single-node SRT supports only native MTP, DSpark or no speculation") + # EAGLE3 covers AgentX configs whose golden curve is the EAGLE3 draft head. + if spec and spec["method"] not in {"eagle", "eagle3", "nextn", "mtp", "dspark"}: + raise ValueError("Single-node SRT supports only native MTP, EAGLE3, DSpark or no speculation") speculation = "mtp" if spec else "none" # Some DSpark configs label the checkpoint's bundled draft as draft_model. if spec and spec["method"] == "dspark" and environment["SPEC_DECODING"] == "draft_model": diff --git a/utils/test_srt_single_node.py b/utils/test_srt_single_node.py index 55011e955d..8ab620191f 100644 --- a/utils/test_srt_single_node.py +++ b/utils/test_srt_single_node.py @@ -219,6 +219,9 @@ def test_trt_binding_keeps_engine_options_and_sets_eval_token_budget(point): ]] with pytest.raises(ValueError, match="moe_expert_parallel_size"): runtime_arguments(f"{path}:base", {**env, "EP_SIZE": "1"}) + recipe["roles"]["agg"]["args"]["speculative_config"] = {"decoding_type": "Eagle3", "max_draft_len": 3} + path.write_text(yaml.safe_dump({"base": recipe})) + runtime_arguments(f"{path}:base", env) def test_atom_binding_uses_allocation_tp_and_native_mtp_arguments(point): From 208ef8adfafacbeadb844029f3625a1a56598dac Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:30:18 -0500 Subject: [PATCH 052/105] feat(agentx): port the DSV4.1 Flash B300 vLLM AgentX config to srt-slurm --- .../dsv41flash/vllm/b300-fp4-mtp/agentic.yaml | 76 +++++++++++++++++++ configs/nvidia-master.yaml | 4 +- 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..a73dafe593 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,76 @@ +# DeepSeek-V4.1-Flash AgentX on B300 with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). Engram weights use +# UVA DRAM; the KV cache is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-b300-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai:deepseekv41-flash-0909 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + engram-config: '{"cpu_offload":true}' + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + max-num-seqs: 256 + disable-uvicorn-access-log: true + env: + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Piecewise graph capture sizes are multiples of the +# six-token DSpark verification block, denser for small batches, and each +# batched-token limit matches the largest captured graph: CONC <= 4 and TP2 +# CONC 128 capture up to 2046 tokens (the latter at 0.97 memory utilization), +# every other point up to 8190. +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}', max-cudagraph-capture-size: 2046, max-num-batched-tokens: 2048}}}, benchmark: {env: {CONC: '1'}}} +override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}', max-cudagraph-capture-size: 2046, max-num-batched-tokens: 2048}}}, benchmark: {env: {CONC: '2'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}', max-cudagraph-capture-size: 2046, max-num-batched-tokens: 2048}}}, benchmark: {env: {CONC: '4'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '8'}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '16'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '32'}}} +override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '64'}}} +override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '128'}}} +override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}', max-cudagraph-capture-size: 2046, max-num-batched-tokens: 2048}}}, benchmark: {env: {CONC: '2'}}} +override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}', max-cudagraph-capture-size: 2046, max-num-batched-tokens: 2048}}}, benchmark: {env: {CONC: '4'}}} +override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '8'}}} +override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '16'}}} +override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '32'}}} +override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '64'}}} +override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}', max-cudagraph-capture-size: 2046, max-num-batched-tokens: 2048, gpu-memory-utilization: 0.97}}}, benchmark: {env: {CONC: '128'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 96467d6f33..7a6aac2fa8 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8428,9 +8428,9 @@ dsv41flash-fp4-b300-vllm-agentic-dspark: search-space: # Engram weights use UVA DRAM; the KV cache stays GPU-resident. The B300 # script uses explicit piecewise CUDA graph capture tiers. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml } # Run the same B300 recipe on two GPUs. - - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [2, 4, 8, 16, 32, 64, 128] } + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml } dsv41flash-fp4-b200-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 From 5e9a32edf76b2c4c0fccc7a9525151e1e1f9dd1c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:30:18 -0500 Subject: [PATCH 053/105] feat(agentx): port the MiniMax-M3 B200 and B300 vLLM AgentX configs to srt-slurm --- .../configs/vllm-simple-kv-offload.sh | 4 + .../minimaxm3/vllm/b200-fp4-mtp/agentic.yaml | 80 +++++++++++++++++++ .../minimaxm3/vllm/b300-fp4-mtp/agentic.yaml | 73 +++++++++++++++++ configs/nvidia-master.yaml | 14 ++-- 4 files changed, 164 insertions(+), 7 deletions(-) create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/vllm-simple-kv-offload.sh create mode 100644 benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml create mode 100644 benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-simple-kv-offload.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-simple-kv-offload.sh new file mode 100755 index 0000000000..208464daa7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-simple-kv-offload.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Split heterogeneous KV backing storage into valid SimpleCPUOffload regions in the worker's vLLM. +set -euo pipefail +python3 /infmax-workspace/runners/patch_vllm_simple_kv_offload.py diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..0c83164aba --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,80 @@ +# MiniMax-M3 NVFP4 AgentX on B200 with vLLM EAGLE3 (the GQA draft head). +# DRAM KV offload uses SimpleCPUOffloadConnector in lazy mode. +base: + schema: 2 + name: minimaxm3-fp4-b200-vllm-agentic + model: + path: hf:nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: nvidia/MiniMax-M3-NVFP4 + gpu-memory-utilization: 0.9 + block-size: 128 + language-model-only: true + enable-prefix-caching: true + no-enable-flashinfer-autotune: true + reasoning-parser: minimax_m3 + tool-call-parser: minimax_m3 + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + kv-cache-dtype: fp8 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + trust-remote-code: true + # Three-token EAGLE3 with the GQA draft head. Throughput runs switch to + # synthetic rejection at the golden acceptance length. + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + env: + PYTHONNOUSERSITE: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/MiniMax-M3-NVFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. DRAM points patch vLLM's SimpleCPUOffload worker for +# heterogeneous layer regions and give it the whole host budget +# (TOTAL_CPU_DRAM_GB GiB) with lazy offload. +override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp4_c5: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '5', KV_OFFLOADING: none}}} +override_tp4_c10: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: none}}} +override_tp4_c15: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '15', KV_OFFLOADING: none}}} +override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: none}}} +override_tp4_c15_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '15', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} +override_tp4_c20_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} +override_tp4_c25_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '25', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} +override_tp4_c30_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '30', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} +override_tp4_c32_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} +override_tp4_c34_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '34', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} +override_tp4_c36_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '36', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} +override_tp4_c38_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '38', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} +override_tp4_c40_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '40', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..d4bc31ace7 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,73 @@ +# MiniMax-M3 NVFP4 AgentX on B300 with vLLM EAGLE3 (the GQA draft head). +# DRAM KV offload uses SimpleCPUOffloadConnector in lazy mode. +base: + schema: 2 + name: minimaxm3-fp4-b300-vllm-agentic + model: + path: hf:nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: nvidia/MiniMax-M3-NVFP4 + gpu-memory-utilization: 0.9 + block-size: 128 + language-model-only: true + enable-prefix-caching: true + no-enable-flashinfer-autotune: true + reasoning-parser: minimax_m3 + tool-call-parser: minimax_m3 + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + kv-cache-dtype: fp8 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + trust-remote-code: true + # Three-token EAGLE3 with the GQA draft head. Throughput runs switch to + # synthetic rejection at the golden acceptance length. + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + env: + PYTHONNOUSERSITE: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/MiniMax-M3-NVFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. DRAM points patch vLLM's SimpleCPUOffload worker for +# heterogeneous layer regions and give it the whole host budget +# (TOTAL_CPU_DRAM_GB GiB) with lazy offload. +override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp4_c5: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '5', KV_OFFLOADING: none}}} +override_tp4_c10: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: none}}} +override_tp4_c15: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '15', KV_OFFLOADING: none}}} +override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: none}}} +override_tp4_c30_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1099511627776,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '30', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1024'}}} +override_tp2_c24_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":804232626176,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '749'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 7a6aac2fa8..35802d915c 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5573,12 +5573,12 @@ minimaxm3-fp4-b300-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.683 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1] } - - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20] } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [30] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [30], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } - dram-utilization: 1.0 search-space: - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24] } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 @@ -5727,11 +5727,11 @@ minimaxm3-fp4-b200-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.683 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1] } - - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml } - dram-utilization: 1.0 search-space: - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [15, 20, 25, 30, 32, 34, 36, 38, 40] } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [15, 20, 25, 30, 32, 34, 36, 38, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml } minimaxm3-fp4-b200-trtllm-agentic-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 model: nvidia/MiniMax-M3-NVFP4 From a5ce1b701278fe1c90190d06d4dbf98711ae319b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:30:18 -0500 Subject: [PATCH 054/105] feat(agentx): port the DSV4 B200 and B300 vLLM AgentX configs to srt-slurm --- .../configs/vllm-router-0.1.14.sh | 8 ++ .../dsv4/vllm/b200-fp4-mtp/agentic.yaml | 87 +++++++++++++++++++ .../dsv4/vllm/b300-fp4-mtp/agentic.yaml | 87 +++++++++++++++++++ configs/nvidia-master.yaml | 14 +-- 4 files changed, 189 insertions(+), 7 deletions(-) create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/vllm-router-0.1.14.sh create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-router-0.1.14.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-router-0.1.14.sh new file mode 100755 index 0000000000..9b372a0478 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-router-0.1.14.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Install the vLLM Router that fronts single-node DP-attention ranks (worker and router containers). +set -euo pipefail +pip_install=(python3 -m pip install) +if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages; then + pip_install+=(--break-system-packages) +fi +"${pip_install[@]}" --quiet vllm-router==0.1.14 diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..36de08edc1 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,87 @@ +# DeepSeek-V4-Pro-0813 AgentX on B200 with vLLM native DSpark (six draft +# tokens). TP8 keeps the KV cache GPU-resident; DEP8 offloads it to host DRAM +# through SimpleCPUOffloadConnector. +base: + schema: 2 + name: dsv4-fp4-b200-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-591bb95 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 720 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + kv-cache-dtype: fp8 + block-size: 256 + max-model-len: 1048576 + gpu-memory-utilization: 0.90 + numa-bind: true + enable-cumem-allocator: true + no-enable-flashinfer-autotune: true + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v4 + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"indexer_kv_dtype":"mxfp4"}' + # Throughput runs switch to synthetic rejection at the golden acceptance length. + speculative-config: '{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"probabilistic"}' + no-disable-hybrid-kv-cache-manager: true + disable-uvicorn-access-log: true + env: + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '32768' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_DSV4_MEGA_FP8_COMBINE: '1' + VLLM_RPC_TIMEOUT: '600000' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + VLLM_FLOAT32_MATMUL_PRECISION: high + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Graphs capture every decode batch of up to +# max-num-seqs sequences in tokens (1 target + 6 drafts each). TP8 admits CONC +# with a fixed 50 GiB KV cache, and also captures piecewise mixed batches of +# 100-500 tokens. DEP8 runs one data-parallel rank per GPU behind a +# consistent-hash vLLM Router (turns of one conversation share a rank), admits +# 2x CONC across the ranks with MegaMoE experts, reserves verification slots +# beyond an 8192-token prefill budget, and splits TOTAL_CPU_DRAM_GB GB of eager +# offload across the ranks; c160 and c192 shrink the per-rank KV cache. +override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, max-num-batched-tokens: 16384, kv-cache-memory-bytes: 53687091200, max-num-seqs: 1, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,100,200,300,400,500]}'}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, max-num-batched-tokens: 16384, kv-cache-memory-bytes: 53687091200, max-num-seqs: 4, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,100,200,300,400,500]}'}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp8_c6: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, max-num-batched-tokens: 16384, kv-cache-memory-bytes: 53687091200, max-num-seqs: 6, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,100,200,300,400,500]}'}}}, benchmark: {env: {CONC: '6', KV_OFFLOADING: none}}} +override_tp8_c10: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, max-num-batched-tokens: 16384, kv-cache-memory-bytes: 53687091200, max-num-seqs: 10, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,100,200,300,400,500]}'}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: none}}} +override_tp8_c14: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, max-num-batched-tokens: 16384, kv-cache-memory-bytes: 53687091200, max-num-seqs: 14, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,100,200,300,400,500]}'}}}, benchmark: {env: {CONC: '14', KV_OFFLOADING: none}}} +override_tp8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, max-num-batched-tokens: 16384, kv-cache-memory-bytes: 53687091200, max-num-seqs: 16, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,100,105,112,200,300,400,500]}'}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: none}}} +override_dep8_c32: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 16, long-prefill-token-threshold: 512, kv-cache-memory-bytes: 37580963840, max-num-batched-tokens: 8248, max-num-seqs: 8, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1947', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_dep8_c64: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 16, long-prefill-token-threshold: 512, kv-cache-memory-bytes: 37580963840, max-num-batched-tokens: 8304, max-num-seqs: 16, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1947', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_dep8_c96: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 16, long-prefill-token-threshold: 512, kv-cache-memory-bytes: 37580963840, max-num-batched-tokens: 8360, max-num-seqs: 24, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '96', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1947', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_dep8_c128: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 16, long-prefill-token-threshold: 512, kv-cache-memory-bytes: 37580963840, max-num-batched-tokens: 8416, max-num-seqs: 32, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '128', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1947', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_dep8_c160: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 16, long-prefill-token-threshold: 512, kv-cache-memory-bytes: 36507222016, max-num-batched-tokens: 8472, max-num-seqs: 40, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224,231,238,245,252,259,266,273,280],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '160', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1947', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_dep8_c192: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 16, long-prefill-token-threshold: 512, kv-cache-memory-bytes: 36507222016, max-num-batched-tokens: 8528, max-num-seqs: 48, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224,231,238,245,252,259,266,273,280,287,294,301,308,315,322,329,336],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '192', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1947', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..b3476ad511 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,87 @@ +# DeepSeek-V4-Pro AgentX on B300 with vLLM native MTP (three draft tokens). +# TP8 and TP4 c8 are GPU-resident; TP4 c16, DEP4 and DEP8 offload KV to host +# DRAM through SimpleCPUOffloadConnector. +base: + schema: 2 + name: dsv4-fp4-b300-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4-Pro + container: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-426e59f + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + trust-remote-code: true + no-enable-flashinfer-autotune: true + no-disable-hybrid-kv-cache-manager: true + kv-cache-dtype: fp8 + block-size: 256 + max-model-len: 1048576 + attention-config: '{"use_fp4_indexer_cache":true,"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}' + # Throughput runs switch to synthetic rejection at the golden acceptance length. + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + disable-uvicorn-access-log: true + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v4 + env: + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '32768' + VLLM_DSV4_MEGA_FP8_COMBINE: '1' + NCCL_NVLS_ENABLE: '1' + VLLM_USE_RUST_FRONTEND: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + VLLM_FLOAT32_MATMUL_PRECISION: high + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Decode graphs capture every batch of up to +# max-num-seqs sequences in tokens (1 target + 3 drafts each). TP points admit +# 2x CONC with the FlashInfer all-reduce; DEP points run one data-parallel rank +# per GPU behind a consistent-hash vLLM Router (turns of one conversation share +# a rank), admit 2x CONC across the ranks with MegaMoE experts, and keep eager +# offload so block hashes agree across ranks. DEP8 takes a larger prefill +# budget and more memory headroom. DRAM points split TOTAL_CPU_DRAM_GB GB +# across the ranks. +override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 2, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8],"mode":0}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto'}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 8, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32],"mode":0}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto'}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 2, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8],"mode":0}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto'}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 4, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16],"mode":0}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto'}}}, benchmark: {env: {CONC: '2', KV_OFFLOADING: none}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 8, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32],"mode":0}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto'}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp4_c6: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 12, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48],"mode":0}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto'}}}, benchmark: {env: {CONC: '6', KV_OFFLOADING: none}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 16, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto'}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 32, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356000000000,"enable_cross_layers_blocks":"true","lazy_offload":true}}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1424'}}} +override_dep4_c48: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 1, data-parallel-size: 4, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 8, long-prefill-token-threshold: 512, max-num-batched-tokens: 8192, gpu-memory-utilization: 0.95, max-num-seqs: 24, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356000000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '48', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1424', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_dep4_c64: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 1, data-parallel-size: 4, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 8, long-prefill-token-threshold: 512, max-num-batched-tokens: 8192, gpu-memory-utilization: 0.95, max-num-seqs: 32, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356000000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1424', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_dep8_c128: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 8, long-prefill-token-threshold: 512, max-num-batched-tokens: 16384, gpu-memory-utilization: 0.92, max-num-seqs: 32, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '128', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '2849', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_dep8_c256: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 8, long-prefill-token-threshold: 512, max-num-batched-tokens: 16384, gpu-memory-utilization: 0.92, max-num-seqs: 64, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '256', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '2849', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_dep8_c384: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 8, long-prefill-token-threshold: 512, max-num-batched-tokens: 16384, gpu-memory-utilization: 0.92, max-num-seqs: 96, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,292,296,300,304,308,312,316,320,324,328,332,336,340,344,348,352,356,360,364,368,372,376,380,384],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '384', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '2849', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_dep8_c512: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 8, long-prefill-token-threshold: 512, max-num-batched-tokens: 16384, gpu-memory-utilization: 0.92, max-num-seqs: 128, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,292,296,300,304,308,312,316,320,324,328,332,336,340,344,348,352,356,360,364,368,372,376,380,384,388,392,396,400,404,408,412,416,420,424,428,432,436,440,444,448,452,456,460,464,468,472,476,480,484,488,492,496,500,504,508,512],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '512', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '2849', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_dep8_c576: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 8, long-prefill-token-threshold: 512, max-num-batched-tokens: 16384, gpu-memory-utilization: 0.92, max-num-seqs: 144, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,292,296,300,304,308,312,316,320,324,328,332,336,340,344,348,352,356,360,364,368,372,376,380,384,388,392,396,400,404,408,412,416,420,424,428,432,436,440,444,448,452,456,460,464,468,472,476,480,484,488,492,496,500,504,508,512,516,520,524,528,532,536,540,544,548,552,556,560,564,568,572,576],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '576', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '2849', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 35802d915c..24b8c174f3 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -948,9 +948,9 @@ dsv4-fp4-b200-vllm-agentic-mtp: - dram-utilization: 0.90 search-space: # TP8 resident KV + DSpark6 - - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 6, 10, 14, 16] } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 6, 10, 14, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml } # DEP8 SimpleCPU + DSpark6 - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [32, 64, 96, 128, 160, 192], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [32, 64, 96, 128, 160, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml } # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 # does not have a B300-specific recipe, so this config reuses the existing DSR1 FP4 @@ -1660,15 +1660,15 @@ dsv4-fp4-b300-vllm-agentic-mtp: - dram-utilization: 0.95 search-space: # TP8 GPU-resident + MTP (num_speculative_tokens=3) - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4] } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml } # TP4 GPU-resident + MTP (num_speculative_tokens=3) - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml } # TP4 SimpleCPU + MTP (num_speculative_tokens=3) - - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [16] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml } # DEP4 SimpleCPU + MTP (num_speculative_tokens=3) - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [48, 64], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [48, 64], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml } # DEP8 SimpleCPU + MTP (num_speculative_tokens=3) - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [128, 256, 384, 512, 576], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [128, 256, 384, 512, 576], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml } qwen3.5-fp8-h200-sglang: image: lmsysorg/sglang:v0.5.14-cu130 From 556e3d0d5a0b73404471285305829e465ecd0644 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:30:33 -0500 Subject: [PATCH 055/105] feat(agentx): port the Qwen3.5 FP4 B200 SGLang AgentX config to srt-slurm --- .../qwen3.5/sglang/b200-fp4-mtp/agentic.yaml | 104 ++++++++++++++++++ configs/nvidia-master.yaml | 8 +- 2 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..900977903a --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,104 @@ +# Qwen3.5-397B-A17B NVFP4 AgentX on B200 with SGLang native NEXTN MTP. The KV +# cache is GPU-resident or extends to host DRAM through HiCache. +base: + schema: 2 + name: qwen3.5-fp4-b200-sglang-agentic + model: + path: hf:nvidia/Qwen3.5-397B-A17B-NVFP4 + container: lmsysorg/sglang:v0.5.16-cu130 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + trust-remote-code: true + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.8 + stream-interval: 50 + scheduler-recv-interval: 10 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-metrics: true + enable-cache-report: true + env: + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + # Outlast AIPerf's 300 s client connection pool. + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/Qwen3.5-397B-A17B-NVFP4 + # 256k-capped corpus, within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC and the graph batch is CONC, +# capped at 64. TP4 tokenizes in parallel to keep 256k warmups within the +# client timeout; TP2 keeps the single tokenizer, whose multi-worker startup +# races with TP2 HiCache shared-memory initialization. HiCache sizes each of +# the target KV and Mamba host pools per rank with a 1 GB/rank alignment +# reserve and the 1/15 NEXTN draft pool: (DRAM - TP) * 15 / TP / 31 GB. +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} +override_tp4_c12: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: none}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 32, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: none}}} +override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 40, cuda-graph-max-bs: 20}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: none}}} +override_tp4_c24: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 48, cuda-graph-max-bs: 24}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: none}}} +override_tp4_c28: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 56, cuda-graph-max-bs: 28}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: none}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 64, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: none}}} +override_tp4_c40: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 80, cuda-graph-max-bs: 40}}}, benchmark: {env: {CONC: '40', KV_OFFLOADING: none}}} +override_tp4_c48: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 96, cuda-graph-max-bs: 48}}}, benchmark: {env: {CONC: '48', KV_OFFLOADING: none}}} +override_tp4_c56: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 112, cuda-graph-max-bs: 56}}}, benchmark: {env: {CONC: '56', KV_OFFLOADING: none}}} +override_tp4_c60: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 120, cuda-graph-max-bs: 60}}}, benchmark: {env: {CONC: '60', KV_OFFLOADING: none}}} +override_tp4_c62: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 124, cuda-graph-max-bs: 62}}}, benchmark: {env: {CONC: '62', KV_OFFLOADING: none}}} +override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 128, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: none}}} +override_tp4_hicache_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 128, cuda-graph-max-bs: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_hicache_c66: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 132, cuda-graph-max-bs: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '66', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_hicache_c68: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 136, cuda-graph-max-bs: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '68', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_hicache_c70: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 140, cuda-graph-max-bs: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '70', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_hicache_c72: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 144, cuda-graph-max-bs: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '72', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} +override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} +override_tp2_c12: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: none}}} +override_tp2_c14: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 28, cuda-graph-max-bs: 14}}}, benchmark: {env: {CONC: '14', KV_OFFLOADING: none}}} +override_tp2_hicache_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 32, cuda-graph-max-bs: 16, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '432'}}} +override_tp2_hicache_c18: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 36, cuda-graph-max-bs: 18, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '18', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '432'}}} +override_tp2_hicache_c20: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 40, cuda-graph-max-bs: 20, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '432'}}} +override_tp2_hicache_c22: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 44, cuda-graph-max-bs: 22, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '22', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '432'}}} +override_tp2_hicache_c24: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 48, cuda-graph-max-bs: 24, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '432'}}} +override_tp2_hicache_c28: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 56, cuda-graph-max-bs: 28, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '432'}}} +override_tp2_hicache_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 64, cuda-graph-max-bs: 32, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '432'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 46179b44a9..01a38f037f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5452,10 +5452,10 @@ qwen3.5-fp4-b200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72] } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14] } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml } # Qwen3.8-Flash-Next NVFP4 AgentX on B200 via SGLang with native NEXTN MTP. From f2b229d299ee9919f374aca1cd22b2f9d28df628 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:30:33 -0500 Subject: [PATCH 056/105] feat(agentx): port the Qwen3.5 FP8 B200 SGLang AgentX config to srt-slurm --- .../qwen3.5/sglang/b200-fp8-mtp/agentic.yaml | 104 ++++++++++++++++++ configs/nvidia-master.yaml | 8 +- 2 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..0b883dde79 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml @@ -0,0 +1,104 @@ +# Qwen3.5-397B-A17B FP8 AgentX on B200 with SGLang native NEXTN MTP. The KV +# cache is GPU-resident or extends to host DRAM through HiCache. +base: + schema: 2 + name: qwen3.5-fp8-b200-sglang-agentic + model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + precision: fp8 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.8 + stream-interval: 50 + scheduler-recv-interval: 10 + # Parallel tokenization keeps 256k warmups within the client timeout. + tokenizer-worker-num: 6 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-metrics: true + enable-cache-report: true + env: + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + # Outlast AIPerf's 300 s client connection pool. + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + # 256k-capped corpus, within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC and the decode graph batch is +# CONC, capped at 64. HiCache sizes each of the target KV and Mamba host pools +# per rank with a 1 GB/rank alignment reserve and the 1/15 NEXTN draft pool: +# (DRAM - TP) * 15 / TP / 31 GB. +override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 2, cuda-graph-max-bs-decode: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 8, cuda-graph-max-bs-decode: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp8_c8: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 16, cuda-graph-max-bs-decode: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} +override_tp8_c12: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 24, cuda-graph-max-bs-decode: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: none}}} +override_tp8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 32, cuda-graph-max-bs-decode: 16}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: none}}} +override_tp8_c20: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 40, cuda-graph-max-bs-decode: 20}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: none}}} +override_tp8_c24: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 48, cuda-graph-max-bs-decode: 24}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: none}}} +override_tp8_c28: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 56, cuda-graph-max-bs-decode: 28}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: none}}} +override_tp8_c32: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 64, cuda-graph-max-bs-decode: 32}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: none}}} +override_tp8_c40: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 80, cuda-graph-max-bs-decode: 40}}}, benchmark: {env: {CONC: '40', KV_OFFLOADING: none}}} +override_tp8_c48: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 96, cuda-graph-max-bs-decode: 48}}}, benchmark: {env: {CONC: '48', KV_OFFLOADING: none}}} +override_tp8_c56: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 112, cuda-graph-max-bs-decode: 56}}}, benchmark: {env: {CONC: '56', KV_OFFLOADING: none}}} +override_tp8_c60: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 120, cuda-graph-max-bs-decode: 60}}}, benchmark: {env: {CONC: '60', KV_OFFLOADING: none}}} +override_tp8_c62: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 124, cuda-graph-max-bs-decode: 62}}}, benchmark: {env: {CONC: '62', KV_OFFLOADING: none}}} +override_tp8_c64: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 128, cuda-graph-max-bs-decode: 64}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: none}}} +override_tp8_hicache_c64: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 128, cuda-graph-max-bs-decode: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} +override_tp8_hicache_c66: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 132, cuda-graph-max-bs-decode: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '66', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} +override_tp8_hicache_c68: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 136, cuda-graph-max-bs-decode: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '68', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} +override_tp8_hicache_c70: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 140, cuda-graph-max-bs-decode: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '70', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} +override_tp8_hicache_c72: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 144, cuda-graph-max-bs-decode: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '72', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} +override_tp8_hicache_c76: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 152, cuda-graph-max-bs-decode: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '76', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 2, cuda-graph-max-bs-decode: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 8, cuda-graph-max-bs-decode: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 16, cuda-graph-max-bs-decode: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} +override_tp4_c12: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 24, cuda-graph-max-bs-decode: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: none}}} +override_tp4_c14: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 28, cuda-graph-max-bs-decode: 14}}}, benchmark: {env: {CONC: '14', KV_OFFLOADING: none}}} +override_tp4_hicache_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 32, cuda-graph-max-bs-decode: 16, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_hicache_c18: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 36, cuda-graph-max-bs-decode: 18, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '18', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_hicache_c20: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 40, cuda-graph-max-bs-decode: 20, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_hicache_c22: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 44, cuda-graph-max-bs-decode: 22, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '22', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_hicache_c24: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 48, cuda-graph-max-bs-decode: 24, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_hicache_c28: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 56, cuda-graph-max-bs-decode: 28, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_hicache_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 64, cuda-graph-max-bs-decode: 32, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 01a38f037f..948df1d591 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1188,10 +1188,10 @@ qwen3.5-fp8-b200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64] } - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32] } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml } qwen3.5-fp4-b200-sglang: image: lmsysorg/sglang:v0.5.19-cu130 From f4c601d59322deb97ff9acf7ae52dc87b0b48829 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:30:33 -0500 Subject: [PATCH 057/105] feat(agentx): port the Qwen3.8 Next FP4 B200 SGLang AgentX config to srt-slurm --- .../sglang/b200-fp4-mtp/agentic.yaml | 70 +++++++++++++++++++ configs/nvidia-master.yaml | 2 +- 2 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..5ca678e34c --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,70 @@ +# Qwen3.8-Flash-Next NVFP4 AgentX on B200 via SGLang with native NEXTN MTP. +# The 126 GiB checkpoint fits on one GPU, so the cookbook command is TP1. +base: + schema: 2 + name: qwen3.8next-fp4-b200-sglang-agentic + model: + path: hf:RadixArk/Qwen3.8-Flash-Next-NVFP4 + container: lmsysorg/sglang:qwen38flashnext + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 1 + args: + served-model-name: RadixArk/Qwen3.8-Flash-Next-NVFP4 + trust-remote-code: true + tensor-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + # Quantization is read from the checkpoint. + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + # Flashinfer linear-attn decode on SM100+ requires a bfloat16 state. + mamba-ssm-dtype: bfloat16 + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + reasoning-parser: auto + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + enable-metrics: true + enable-cache-report: true + env: + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + # Outlast AIPerf's 300 s client connection pool. + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: RadixArk/Qwen3.8-Flash-Next-NVFP4 + # 256k-capped corpus, within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. NEXTN resets an unset admission cap to 48, so it is +# explicit: 2x CONC, with the graph batch equal to it. C16 needs a larger +# static fraction and a bounded Mamba cache. +override_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 2, mem-fraction-static: 0.8}}}, benchmark: {env: {CONC: '1'}}} +override_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 8, mem-fraction-static: 0.8}}}, benchmark: {env: {CONC: '4'}}} +override_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 16, mem-fraction-static: 0.8}}}, benchmark: {env: {CONC: '8'}}} +override_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 32, mem-fraction-static: 0.9, max-mamba-cache-size: 160}}}, benchmark: {env: {CONC: '16'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 948df1d591..1f66b1257d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5472,7 +5472,7 @@ qwen3.8next-fp4-b200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16] } + - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml } qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 From c084079bebf9bd8c1185b5355aca34757966290c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:30:33 -0500 Subject: [PATCH 058/105] fix(h100): point native single-node uv caches at shared NFS --- runners/launch_h100-dgxc-slurm.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 31058b0cc2..0bbbd84289 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -26,6 +26,9 @@ if [[ "$EXECUTION_PATH" == native-single-node ]]; then HF_HUB_CACHE_MOUNT=/mnt/nfs/sa-shared/gharunners/hf-hub-cache SRT_MODEL_PATH="hf:$MODEL" SRT_SQUASH_FILE="/mnt/nfs/lustre/containers/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + # The job's host-side orchestrator cannot write the compute node's home. + export UV_CACHE_DIR="/mnt/nfs/sa-shared/.uv/cache" + export UV_PYTHON_INSTALL_DIR="/mnt/nfs/sa-shared/.uv/python" launch_srt_single_node h100-dgxc-slurm \ --var SLURM_ACCOUNT "$SLURM_ACCOUNT" --var SLURM_PARTITION "$SLURM_PARTITION" \ --var CONTAINER_KEY "$IMAGE" From b6d744fe7d48e366080c33e71b9ab78c99a5a00f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:30:34 -0500 Subject: [PATCH 059/105] feat(agentx): port the GLM-5.2 FP4 B200 SGLang AgentX config to srt-slurm --- .../glm5.2/sglang/b200-fp4-mtp/agentic.yaml | 79 +++++++++++++++++++ configs/nvidia-master.yaml | 2 +- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..05f1568b50 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,79 @@ +# GLM-5.2 NVFP4 AgentX on B200 with SGLang EAGLE MTP off the checkpoint's nextn +# head: the TP8 low-latency arm (fp8 KV, cutedsl bf16 GEMM) with HiCache +# host-DRAM offload at every point. +base: + schema: 2 + name: glm5.2-fp4-b200-sglang-agentic + model: + path: hf:nvidia/GLM-5.2-NVFP4 + container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 1 + kv-cache-dtype: fp8_e4m3 + bf16-gemm-backend: cutedsl + max-prefill-tokens: 8192 + quantization: modelopt_fp4 + # GLM-5.2 emits the GLM-4.7 tool-call format; glm45 leaves calls as raw text. + tool-call-parser: glm47 + reasoning-parser: glm45 + chunked-prefill-size: 8192 + # B200 HBM leaves ~31 GB outside the static share for EAGLE verification, + # graph capture and DSA indexer temporaries. + mem-fraction-static: 0.83 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + watchdog-timeout: 1800 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + # Keep FlashInfer/Triton/Inductor/JIT caches off the shared home. + SGLANG_CACHE_DIR: /tmp/sglang-cache + # Outlast AIPerf's pooled keep-alive connections. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/GLM-5.2-NVFP4 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +# One variant per point. Admission is 2x CONC and the graph batch matches it, +# capped at 64. MLA ranks hold complete per-token KV, so ratio 0.75 gives too +# few host slots past C8; C12/C16 pin a 169 GB/rank pool (~1,662 GB across TP8 +# with the DSA indexer, within the 1,731 GB budget). +override_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 2, hicache-ratio: 0.75}}}, benchmark: {env: {CONC: '1'}}} +override_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 8, hicache-ratio: 0.75}}}, benchmark: {env: {CONC: '4'}}} +override_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 16, hicache-ratio: 0.75}}}, benchmark: {env: {CONC: '8'}}} +override_c12: {roles: {agg: {args: {max-running-requests: 24, cuda-graph-max-bs: 24, hicache-size: 169}}}, benchmark: {env: {CONC: '12'}}} +override_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 32, hicache-size: 169}}}, benchmark: {env: {CONC: '16'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 1f66b1257d..2f647b9687 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7155,7 +7155,7 @@ glm5.2-fp4-b200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml } # First GLM-5.2 FP8 AgentX recipe on B200, the FP8 precision sibling of # glm5.2-fp4-b200-sglang-agentic-mtp. Same spec-decode-only shape per the AgentX From 71c94daeadc3caef6ce56fbf521ca71e59921fb0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:30:34 -0500 Subject: [PATCH 060/105] feat(agentx): port the GLM-5.2 FP8 B200 SGLang AgentX config to srt-slurm --- .../glm5.2/sglang/b200-fp8-mtp/agentic.yaml | 79 +++++++++++++++++++ configs/nvidia-master.yaml | 2 +- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..88d37041c1 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml @@ -0,0 +1,79 @@ +# GLM-5.2 FP8 AgentX on B200 with SGLang EAGLE MTP off the checkpoint's nextn +# head: the TP8 low-latency arm (fp8 KV, cutedsl bf16 GEMM) with HiCache +# host-DRAM offload at every point. +base: + schema: 2 + name: glm5.2-fp8-b200-sglang-agentic + model: + path: hf:zai-org/GLM-5.2-FP8 + container: lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b + precision: fp8 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 1 + kv-cache-dtype: fp8_e4m3 + bf16-gemm-backend: cutedsl + max-prefill-tokens: 8192 + quantization: fp8 + # GLM-5.2 emits the GLM-4.7 tool-call format; glm45 leaves calls as raw text. + tool-call-parser: glm47 + reasoning-parser: glm45 + chunked-prefill-size: 8192 + # B200 HBM leaves ~31 GB outside the static share for EAGLE verification, + # graph capture and DSA indexer temporaries. + mem-fraction-static: 0.83 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + watchdog-timeout: 1800 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + # Keep FlashInfer/Triton/Inductor/JIT caches off the shared home. + SGLANG_CACHE_DIR: /tmp/sglang-cache + # Outlast AIPerf's pooled keep-alive connections. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: zai-org/GLM-5.2-FP8 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +# One variant per point. Admission is 2x CONC and the graph batch matches it, +# capped at 64. MLA ranks hold complete per-token KV, so ratio 0.75 gives too +# few host slots past C8; C12/C16 pin a 169 GB/rank pool (~1,662 GB across TP8 +# with the DSA indexer, within the 1,731 GB budget). +override_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 2, hicache-ratio: 0.75}}}, benchmark: {env: {CONC: '1'}}} +override_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 8, hicache-ratio: 0.75}}}, benchmark: {env: {CONC: '4'}}} +override_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 16, hicache-ratio: 0.75}}}, benchmark: {env: {CONC: '8'}}} +override_c12: {roles: {agg: {args: {max-running-requests: 24, cuda-graph-max-bs: 24, hicache-size: 169}}}, benchmark: {env: {CONC: '12'}}} +override_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 32, hicache-size: 169}}}, benchmark: {env: {CONC: '16'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 2f647b9687..19fa633701 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7185,7 +7185,7 @@ glm5.2-fp8-b200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml } # GLM-5.2 NVFP4 B200 AgentX on Dynamo + SGLang. EAGLE uses the model's # built-in nextn head, with acceptance pinned to the golden thinking-on AL in From 2166e4c92087d92646bb3675dc2f5c00edc6b8e8 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:30:34 -0500 Subject: [PATCH 061/105] feat(agentx): port the DSV4 FP4 B200 SGLang AgentX config to srt-slurm --- .../dsv4/sglang/b200-fp4-mtp/agentic.yaml | 90 +++++++++++++++++++ configs/nvidia-master.yaml | 6 +- 2 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..5f317e6ebf --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,90 @@ +# DeepSeek-V4-Pro-0813 FP4 AgentX on B200 with SGLang DSpark (block 6). TP8 +# covers the low-concurrency points (GPU KV, then HiCache); DEP8 attention-DP +# fronted by the SGLang Model Gateway covers the high-concurrency HiCache points. +base: + schema: 2 + name: dsv4-fp4-b200-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 + container: lmsysorg/sglang:v0.5.19-cu130 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tensor-parallel-size: 8 + enable-deepseek-v4-fp4-indexer: true + disable-flashinfer-autotune: true + tool-call-parser: deepseekv4 + reasoning-parser: deepseek-v4 + chat-template: /infmax-workspace/benchmarks/single_node/chat_templates/deepseek_v4_thinking.jinja + watchdog-timeout: 1800 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + # The checkpoint lives on Lustre: prefetch sequentially across local + # ranks so post-load repacking reads from page cache. + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + enable-metrics: true + enable-cache-report: true + env: + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + # Outlast AIPerf's pooled keep-alive connections. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_USE_JIT_NORM: '1' + SGLANG_OPT_USE_JIT_INDEXER_METADATA: '1' + SGLANG_OPT_USE_TOPK_V2: '1' + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + # The image's first ptxas on the legacy search path. + TRITON_PTXAS_PATH: /usr/local/cuda-13.0/bin/ptxas + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point; admission is 2x CONC. TP8 captures graphs up to the +# admission cap and runs the FlashInfer MXFP4 MoE; its replicated HiCache +# pools fit ratio 2.75 (~121 GiB/rank) on 1.7 TiB hosts. DEP8 shards the host +# pools (ratio 8), caps graphs at 32, runs MegaMoE with 6144 prefill tokens per +# DP rank, and balances DP admission; the gateway routes sessions by AIPerf +# correlation ID. C160 interleaves prefill more often and bounds the gateway's +# load imbalance. +override_tp8_c1: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 2, cuda-graph-max-bs: 2}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} +override_tp8_c2: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 4, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '2', KV_OFFLOADING: none}}} +override_tp8_c3: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 6, cuda-graph-max-bs: 6}}}, benchmark: {env: {CONC: '3', KV_OFFLOADING: none}}} +override_tp8_c4: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 8, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp8_c5: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 10, cuda-graph-max-bs: 10}}}, benchmark: {env: {CONC: '5', KV_OFFLOADING: none}}} +override_tp8_hicache_c8: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 16, cuda-graph-max-bs: 16, enable-hierarchical-cache: true, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, hicache-ratio: 2.75}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} +override_tp8_hicache_c10: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 20, cuda-graph-max-bs: 20, enable-hierarchical-cache: true, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, hicache-ratio: 2.75}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} +override_tp8_hicache_c16: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 32, cuda-graph-max-bs: 32, enable-hierarchical-cache: true, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, hicache-ratio: 2.75}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} +override_dep8_hicache_c64: {frontend: {type: sglang-router, args: {policy: cache_aware, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, expert-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, incremental-streaming-output: true, stream-interval: 20, tokenizer-worker-num: 8, load-balance-method: total_requests, load-snapshot-publish-interval: 1, prefill-decode-interval: 24, moe-a2a-backend: megamoe, enable-w4a4-mxfp4-megamoe: true, disable-shared-experts-fusion: true, mem-fraction-static: 0.88, swa-full-tokens-ratio: 0.02, chunked-prefill-size: 49152, max-running-requests: 128, cuda-graph-max-bs: 32, enable-hierarchical-cache: true, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, hicache-ratio: 8}, env: {SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320'}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'}}} +override_dep8_hicache_c96: {frontend: {type: sglang-router, args: {policy: cache_aware, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, expert-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, incremental-streaming-output: true, stream-interval: 20, tokenizer-worker-num: 8, load-balance-method: total_requests, load-snapshot-publish-interval: 1, prefill-decode-interval: 24, moe-a2a-backend: megamoe, enable-w4a4-mxfp4-megamoe: true, disable-shared-experts-fusion: true, mem-fraction-static: 0.88, swa-full-tokens-ratio: 0.02, chunked-prefill-size: 49152, max-running-requests: 192, cuda-graph-max-bs: 32, enable-hierarchical-cache: true, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, hicache-ratio: 8}, env: {SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320'}}}, benchmark: {env: {CONC: '96', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'}}} +override_dep8_hicache_c128: {frontend: {type: sglang-router, args: {policy: cache_aware, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, expert-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, incremental-streaming-output: true, stream-interval: 20, tokenizer-worker-num: 8, load-balance-method: total_requests, load-snapshot-publish-interval: 1, prefill-decode-interval: 24, moe-a2a-backend: megamoe, enable-w4a4-mxfp4-megamoe: true, disable-shared-experts-fusion: true, mem-fraction-static: 0.88, swa-full-tokens-ratio: 0.02, chunked-prefill-size: 49152, max-running-requests: 256, cuda-graph-max-bs: 32, enable-hierarchical-cache: true, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, hicache-ratio: 8}, env: {SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320'}}}, benchmark: {env: {CONC: '128', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'}}} +override_dep8_hicache_c160: {frontend: {type: sglang-router, args: {policy: cache_aware, balance-abs-threshold: 32, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, expert-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, incremental-streaming-output: true, stream-interval: 20, tokenizer-worker-num: 8, load-balance-method: total_requests, load-snapshot-publish-interval: 1, prefill-decode-interval: 20, moe-a2a-backend: megamoe, enable-w4a4-mxfp4-megamoe: true, disable-shared-experts-fusion: true, mem-fraction-static: 0.88, swa-full-tokens-ratio: 0.02, chunked-prefill-size: 49152, max-running-requests: 320, cuda-graph-max-bs: 32, enable-hierarchical-cache: true, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, hicache-ratio: 8}, env: {SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320'}}}, benchmark: {env: {CONC: '160', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 19fa633701..d5bcbc3c8a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -931,9 +931,9 @@ dsv4-fp4-b200-sglang-agentic-hicache-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 3, 4, 5] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [8, 10, 16] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [64, 96, 128, 160], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 3, 4, 5], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [8, 10, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [64, 96, 128, 160], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml } dsv4-fp4-b200-vllm-agentic-mtp: image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-591bb95 From a37f73c7d5144e08e937d094a6fd5bb4f383865d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:30:34 -0500 Subject: [PATCH 062/105] feat(agentx): port the MiniMax-M3 FP4 B200 TRT-LLM AgentX config to srt-slurm --- .../configs/minimaxm3-trtllm-rc23.sh | 15 +++ .../trtllm/b200-fp4-mtp/agentic.yaml | 115 ++++++++++++++++++ configs/nvidia-master.yaml | 4 +- 3 files changed, 132 insertions(+), 2 deletions(-) create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-trtllm-rc23.sh create mode 100644 benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-trtllm-rc23.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-trtllm-rc23.sh new file mode 100755 index 0000000000..ed18542140 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-trtllm-rc23.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Patch the worker's TensorRT-LLM 1.3.0rc23 for MiniMax-M3 AgentX: keep +# Prometheus request/iteration metrics without the per-step timing collector, +# and accept BFCL's standard store=false chat field. Every MPI rank runs this +# in the node's container, so serialize and skip what is already applied. +set -eo pipefail +ws=/infmax-workspace +exec 9>/tmp/minimaxm3-trtllm-rc23.lock +flock 9 +IS_AGENTIC=0 SCENARIO_TYPE='' source "$ws/benchmarks/benchmark_lib.sh" +py_executor="$(python3 -c 'from importlib.util import find_spec; from pathlib import Path; print(Path(find_spec("tensorrt_llm").origin).parent)')/_torch/pyexecutor/py_executor.py" +if ! grep -Fq "enabled=False)" "$py_executor"; then + disable_trtllm_detailed_perf_metrics +fi +python3 "$ws/runners/patch_trtllm_chat_store.py" diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..988a828038 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,115 @@ +# MiniMax-M3 NVFP4 AgentX on B200 with TRT-LLM EAGLE3-GQA. DRAM KV offload +# uses TRT-LLM's native secondary pool, pinned per topology rather than derived +# from TOTAL_CPU_DRAM_GB. +base: + schema: 2 + name: minimaxm3-fp4-b200-trtllm-agentic + model: + path: hf:nvidia/MiniMax-M3-NVFP4 + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: trtllm_serve + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: trtllm + served_model_name: nvidia/MiniMax-M3-NVFP4 + # Keep Prometheus without rc23's per-step timing collector and accept BFCL's + # store=false field. + setup_script: minimaxm3-trtllm-rc23.sh + roles: + agg: + nodes: 1 + workers: 1 + # Server-layer flag; the local checkpoint mounts at /model. + extra_args: [--chat_template, /model/chat_template.jinja] + args: + moe_expert_parallel_size: 1 + max_seq_len: 1048576 + max_num_tokens: 16384 + cuda_graph_config: + enable_padding: true + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: [1, 512, 1024, 2048] + enable_userbuffers: true + max_num_streams: 3 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + free_gpu_memory_fraction: 0.94 + enable_block_reuse: true + block_reuse_policy: per_conversation + tokens_per_block: 128 + use_kv_cache_manager_v2: true + dtype: fp8 + event_buffer_max_size: 0 + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + enable_chunked_prefill: true + enable_autotuner: true + trust_remote_code: true + reasoning_parser: minimax_m3 + stream_interval: 20 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: true + num_postprocess_workers: 8 + enable_attention_dp: false + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TLLM_PROFILE_LOG_RANKS: all + PYTHONNOUSERSITE: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TQDM_DISABLE: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/MiniMax-M3-NVFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + KV_OFFLOADING: dram + +# One variant per point. The batch cap is CONC; graphs capture every batch up +# to 20 and a thinned ladder above. The host pool is 250 GiB at TP4 and +# 200 GiB at TP8. +override_tp4_c5: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 5, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '5', TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_c10: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 10, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '10', TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_c15: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 15, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '15', TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 20, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '20', TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_c25: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 25, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 19, 21, 23, 25]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '25', TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_c30: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 30, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 18, 20, 22, 24, 27, 30]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '30', TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_c35: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 35, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 17, 20, 23, 26, 29, 32, 35]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '35', TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_c40: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 40, cuda_graph_config: {batch_sizes: [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '40', TOTAL_CPU_DRAM_GB: '865'}}} +override_tp4_c45: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 45, cuda_graph_config: {batch_sizes: [2, 4, 6, 8, 10, 12, 14, 16, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '45', TOTAL_CPU_DRAM_GB: '865'}}} +override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor_parallel_size: 8, max_batch_size: 1, cuda_graph_config: {batch_sizes: [1]}, kv_cache_config: {host_cache_size: 214748364800}}}}, benchmark: {env: {CONC: '1', TOTAL_CPU_DRAM_GB: '1731'}}} +override_tp8_c5: {roles: {agg: {gpus: 8, args: {tensor_parallel_size: 8, max_batch_size: 5, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5]}, kv_cache_config: {host_cache_size: 214748364800}}}}, benchmark: {env: {CONC: '5', TOTAL_CPU_DRAM_GB: '1731'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index d5bcbc3c8a..dd15134394 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5747,8 +5747,8 @@ minimaxm3-fp4-b200-trtllm-agentic-mtp: # does not size the pool. - dram-utilization: 0.8 search-space: - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [5, 10, 15, 20, 25, 30, 35, 40, 45] } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5] } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [5, 10, 15, 20, 25, 30, 35, 40, 45], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml } # Run the GB200 aggregate configuration with TP4 resident, TP8 resident across # two NVL4 nodes, and TP4 lazy SimpleCPU KV offload variants. The pinned # runtime uses Dynamo-native MiniMax parsing on the worker without the vLLM From aee6c295acdbea0302739d3591b9e67d50f07307 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:21:09 -0500 Subject: [PATCH 063/105] feat(agentx): let srt-slurm AgentX recipes set the AIPerf benchmark grace period The GLM-5.2 MI325X legacy script bounded the post-window drain with --benchmark-grace-period 1800. AIPERF_BENCHMARK_GRACE_PERIOD in a recipe's benchmark env passes it through the shared client. --- benchmarks/srt_agentic.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchmarks/srt_agentic.sh b/benchmarks/srt_agentic.sh index 1352ba010c..09aedbc1b9 100644 --- a/benchmarks/srt_agentic.sh +++ b/benchmarks/srt_agentic.sh @@ -172,6 +172,10 @@ for index in "${!CONCURRENCIES[@]}"; do if [[ "${AIPERF_APPLY_CHAT_TEMPLATE:-}" == true ]]; then REPLAY_CMD+=" --apply-chat-template" fi + # Bounded post-window drain for long responses admitted near the end. + if [[ -n "${AIPERF_BENCHMARK_GRACE_PERIOD:-}" ]]; then + REPLAY_CMD+=" --benchmark-grace-period $AIPERF_BENCHMARK_GRACE_PERIOD" + fi run_agentic_replay_and_write_outputs "$RESULT_DIR" if [ "$index" -lt "$(( ${#CONCURRENCIES[@]} - 1 ))" ]; then From ebdd4e6db1f97278746269065e91eb3780025942 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:32:39 -0500 Subject: [PATCH 064/105] feat(agentx): port the DSV4.1 Flash GB200 and GB300 vLLM AgentX configs to srt-slurm --- .../vllm/gb200-fp4-mtp/agentic.yaml | 76 +++++++++++++++++++ .../vllm/gb300-fp4-mtp/agentic.yaml | 76 +++++++++++++++++++ configs/nvidia-master.yaml | 8 +- 3 files changed, 156 insertions(+), 4 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..6faf35c264 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml @@ -0,0 +1,76 @@ +# DeepSeek-V4.1-Flash AgentX on GB200 with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). Engram weights use +# UVA DRAM; the KV cache is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-gb200-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 + precision: fp4 + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + engram-config: '{"cpu_offload":true}' + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + disable-uvicorn-access-log: true + env: + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Graph capture starts at 64 tokens and doubles until it +# covers CONC x (1 + 5 drafts), up to 2048. TP2 leaves ~145 GiB of weights on +# each 256 GiB GPU, so it caps batched tokens at 4096 (the indexer's 1M-wide +# logits buffer), bounds the scheduler at 2x CONC within [16, 256] (FlashInfer +# autotune fails below 16) and stops capturing above 512 tokens. +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '1'}}} +override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '2'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '4'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '8'}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 128}}}, benchmark: {env: {CONC: '16'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 256}}}, benchmark: {env: {CONC: '32'}}} +override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '64'}}} +override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 1024}}}, benchmark: {env: {CONC: '128'}}} +override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '1'}}} +override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '2'}}} +override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '4'}}} +override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} +override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 128, max-num-batched-tokens: 4096, max-num-seqs: 32}}}, benchmark: {env: {CONC: '16'}}} +override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 256, max-num-batched-tokens: 4096, max-num-seqs: 64}}}, benchmark: {env: {CONC: '32'}}} +override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 128}}}, benchmark: {env: {CONC: '64'}}} +override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 256}}}, benchmark: {env: {CONC: '128'}}} diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..0f9e3d5c26 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml @@ -0,0 +1,76 @@ +# DeepSeek-V4.1-Flash AgentX on GB300 with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). Engram weights use +# UVA DRAM; the KV cache is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-gb300-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai:nightly-af1c01499b289be555c475669ba50a88e96d846e + precision: fp4 + resources: + gpu_type: gb300 + gpus_per_node: 4 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 720 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + engram-config: '{"cpu_offload":true}' + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + disable-uvicorn-access-log: true + env: + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Graph capture starts at 64 tokens and doubles until it +# covers CONC x (1 + 5 drafts), up to 2048. TP2 leaves ~175 GiB of weights on +# each 277 GiB GPU, so it caps batched tokens at 4096 (the indexer's 1M-wide +# logits buffer), bounds the scheduler at 2x CONC within [16, 256] (FlashInfer +# autotune fails below 16) and stops capturing above 512 tokens. +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '1'}}} +override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '2'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '4'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '8'}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 128}}}, benchmark: {env: {CONC: '16'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 256}}}, benchmark: {env: {CONC: '32'}}} +override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '64'}}} +override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 1024}}}, benchmark: {env: {CONC: '128'}}} +override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '1'}}} +override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '2'}}} +override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '4'}}} +override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} +override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 128, max-num-batched-tokens: 4096, max-num-seqs: 32}}}, benchmark: {env: {CONC: '16'}}} +override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 256, max-num-batched-tokens: 4096, max-num-seqs: 64}}}, benchmark: {env: {CONC: '32'}}} +override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 128}}}, benchmark: {env: {CONC: '64'}}} +override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 256}}}, benchmark: {env: {CONC: '128'}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 24b8c174f3..e6b693c869 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8381,13 +8381,13 @@ dsv41flash-fp4-gb200-vllm-agentic-dspark: - dram-utilization: 0.80 search-space: # Engram weights use UVA DRAM; the KV cache stays GPU-resident. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml } # TP2 halves the GPU count per replica. Weights rise to ~145 GiB on each # 256 GiB GPU with the Engram tables in pinned host DRAM, so the arm # takes the same caps as the B200 TP2 arm: batched tokens 4096 (the # indexer's logits buffer is 32 GiB at the upstream 16384) and graph # capture stopped at 512, leaving ~49 GiB of KV per GPU. - - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml } # SGLang arm for DeepSeek-V4.1-Flash AgentX on GB200, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). @@ -8485,13 +8485,13 @@ dsv41flash-fp4-gb300-vllm-agentic-dspark: - dram-utilization: 0.80 search-space: # Engram weights use UVA DRAM; the KV cache stays GPU-resident. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml } # TP2 halves the GPU count per replica. Weights rise to ~175 GiB on each # 277 GiB GPU with the Engram tables in pinned host DRAM, so the arm # takes the same caps as the B200 TP2 arm: batched tokens 4096 (the # indexer's logits buffer is 32 GiB at the upstream 16384) and graph # capture stopped at 512, leaving ~36 GiB of KV per GPU. - - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml } # H200 AgentX arm for DeepSeek-V4.1-Flash. Upstream marks h200 verified and says # the GB200 NVL4 TP4 layout becomes TP8 on 8-GPU nodes, so this is TP8. From 0ebfdadea6161cfec29dffbb8c03428933f46dbc Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:32:47 -0500 Subject: [PATCH 065/105] feat(agentx): port the MiniMax-M3 MI300X vLLM AgentX config to srt-slurm The LMCache DRAM point starts one MP server per TP rank from a setup script inside the worker container, as the legacy script did. --- .../configs/lmcache-mp-rocm.sh | 46 +++++++++++ .../vllm/mi300x-fp8-mtp/agentic.yaml | 82 +++++++++++++++++++ configs/amd-master.yaml | 4 +- 3 files changed, 130 insertions(+), 2 deletions(-) create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/lmcache-mp-rocm.sh create mode 100644 benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/lmcache-mp-rocm.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/lmcache-mp-rocm.sh new file mode 100755 index 0000000000..a69ec274b1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/lmcache-mp-rocm.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Start one LMCache MP server per TP rank in the worker container before vLLM, +# as the legacy MI300X MiniMax-M3 AgentX script did. A variant opts in with +# LMCACHE_SHARDS and LMCACHE_L1_SHARD_GB; its kv-transfer-config lists +# tcp://127.0.0.1:5555 through 5555 + LMCACHE_SHARDS - 1. +set -euo pipefail +[[ -n "${LMCACHE_SHARDS:-}" ]] || exit 0 +: "${LMCACHE_L1_SHARD_GB:?}" +version=0.5.3 +pip_install=(python3 -m pip install) +if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages; then + pip_install+=(--break-system-packages) +fi +"${pip_install[@]}" --quiet --no-cache-dir --no-deps \ + "sortedcontainers==2.4.0" \ + "opentelemetry-exporter-prometheus==0.61b0" \ + "cupy-rocm-7-0==14.1.1" \ + "lmcache==${version}" \ + --find-links "https://github.com/LMCache/LMCache/releases/expanded_assets/v${version}-rocm" +python3 -c "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" + +pids=() +for ((shard = 0; shard < LMCACHE_SHARDS; shard++)); do + # Detached so the servers outlive this preamble and serve the vLLM step. + setsid lmcache server \ + --host 127.0.0.1 --port $((5555 + shard)) \ + --http-host 127.0.0.1 --http-port $((8080 + shard)) \ + --l1-size-gb "$LMCACHE_L1_SHARD_GB" --l1-init-size-gb 10 \ + --l1-read-ttl-seconds 7200 --chunk-size 256 --max-workers 2 \ + --eviction-policy LRU --supported-transfer-mode lmcache_driven \ + > "/logs/lmcache_server_${shard}.log" 2>&1 < /dev/null & + pids+=($!) +done +for ((shard = 0; shard < LMCACHE_SHARDS; shard++)); do + for ((attempt = 0; ; attempt++)); do + python3 -c 'import sys, urllib.request; urllib.request.urlopen(sys.argv[1], timeout=2)' \ + "http://127.0.0.1:$((8080 + shard))/healthcheck" 2> /dev/null && break + if ! kill -0 "${pids[$shard]}" 2>/dev/null || (( attempt >= 600 )); then + echo "ERROR: LMCache server $shard did not become ready" >&2 + tail -n 50 "/logs/lmcache_server_${shard}.log" >&2 || true + exit 1 + fi + sleep 1 + done +done +echo "LMCache: ${LMCACHE_SHARDS} servers ready, ${LMCACHE_L1_SHARD_GB} GB L1 each" diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..21f2ade6b9 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml @@ -0,0 +1,82 @@ +# MiniMax-M3 MXFP8 AgentX on MI300X with vLLM EAGLE3 (GQA draft). The KV cache +# is GPU-resident, or backed by LMCache MP DRAM servers at the offload point. +base: + schema: 2 + name: minimaxm3-fp8-mi300x-vllm-agentic + model: + path: hf:MiniMaxAI/MiniMax-M3-MXFP8 + container: vllm/vllm-openai-rocm:v0.29.0 + precision: fp8 + resources: + gpu_type: mi300x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + # Starts the LMCache servers for the variant that sets LMCACHE_SHARDS. + setup_script: lmcache-mp-rocm.sh + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: MiniMaxAI/MiniMax-M3-MXFP8 + tensor-parallel-size: 8 + trust-remote-code: true + block-size: 128 + gpu-memory-utilization: 0.90 + # MiniMaxM3SparseForConditionalGeneration is not torch-compiled, so + # without breakable graphs the default FULL_AND_PIECEWISE mode aborts. + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + enable-chunked-prefill: true + max-num-batched-tokens: 16384 + language-model-only: true + enable-prefix-caching: true + attention-backend: TRITON_ATTN + kv-cache-dtype: fp8 + tool-call-parser: minimax_m3 + reasoning-parser: minimax_m3 + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + stream-interval: 20 + # Three-token EAGLE3; throughput runs replace verification with the + # golden acceptance length. + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"TRITON_ATTN"}' + env: + PYTHONNOUSERSITE: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + VLLM_ROCM_USE_AITER: '1' + VLLM_ROCM_USE_AITER_MHA: '0' + TORCH_BLAS_PREFER_HIPBLASLT: '1' + NCCL_MIN_NCHANNELS: '112' + GPU_MAX_HW_QUEUES: '2' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: MiniMaxAI/MiniMax-M3-MXFP8 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. Admission is 2x CONC. The DRAM point splits its +# 1298 GB budget into one LMCache L1 shard per TP rank (1298 / 8 = 162 GB). +override_tp8_c2: {roles: {agg: {args: {max-num-seqs: 4}}}, benchmark: {env: {CONC: '2', KV_OFFLOADING: 'none'}}} +override_tp8_c4: {roles: {agg: {args: {max-num-seqs: 8}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: 'none'}}} +override_tp8_c6: {roles: {agg: {args: {max-num-seqs: 12}}}, benchmark: {env: {CONC: '6', KV_OFFLOADING: 'none'}}} +override_tp8_c8: {roles: {agg: {args: {max-num-seqs: 16}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: 'none'}}} +override_tp8_c10: {roles: {agg: {args: {max-num-seqs: 20}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: 'none'}}} +override_tp8_c16_lmcache: {roles: {agg: {args: {max-num-seqs: 32, kv-transfer-config: '{"kv_connector":"LMCacheMPConnector","kv_connector_module_path":"lmcache.integration.vllm.lmcache_mp_connector","kv_role":"kv_both","kv_connector_extra_config":{"lmcache.mp.server_urls":"tcp://127.0.0.1:5555,tcp://127.0.0.1:5556,tcp://127.0.0.1:5557,tcp://127.0.0.1:5558,tcp://127.0.0.1:5559,tcp://127.0.0.1:5560,tcp://127.0.0.1:5561,tcp://127.0.0.1:5562","lmcache.mp.mq_timeout":6000.0}}'}, env: {LMCACHE_SHARDS: '8', LMCACHE_L1_SHARD_GB: '162'}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '1298'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 56a9ae3f57..febbc02db8 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1009,8 +1009,8 @@ minimaxm3-fp8-mi300x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 6, 8, 10] } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [16] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 6, 8, 10], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml } # DeepSeek-V4.1-Flash AgentX on MI300X (gfx942), a copy of the validated MI355X vLLM # arm (run 34710937012) on the same ROCm nightly. gfx942 is not in the upstream From 1a53f2b40561f03fb52b8521d0d4942247964666 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:33:49 -0500 Subject: [PATCH 066/105] feat(agentx): port the DSV4.1 Flash MI355X vLLM AgentX config to srt-slurm --- .../vllm/mi355x-fp4-mtp/agentic.yaml | 93 +++++++++++++++++++ configs/amd-master.yaml | 4 +- 2 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..d6bf50a85d --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,93 @@ +# DeepSeek-V4.1-Flash AgentX on MI355X with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). The KV cache is +# GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-mi355x-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai-rocm:nightly-rocm100-7f1a5398e9610d96c473931a26c0e12bbe0d0423 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + # The plain name lets the CK a8w4 MoE kernel win vLLM's priority list; + # aiter_triton_mxfp4_bf16 would force the W4A16 Triton kernel. + moe-backend: aiter + gpu-memory-utilization: 0.9 + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. Adaptive + # verification trims requests on device, which the ROCm DeepSeek-V4 + # indexer does not support. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + # The ROCm sparse SWA path lacks the replay window clamp; bounded replay + # memory-faulted at the first prefix hit with a replay start. + no-swa-bounded-replay: true + disable-uvicorn-access-log: true + env: + VLLM_ROCM_USE_AITER: '1' + VLLM_ROCM_USE_AITER_MOE: '1' + AITER_TRITON_LOG_LEVEL: ERROR + # DeepseekV41ForCausalLM is not torch-compiled upstream; breakable + # graphs keep FULL_AND_PIECEWISE capture working. + VLLM_USE_BREAKABLE_CUDAGRAPH: '1' + OMP_NUM_THREADS: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Full-context corpus for this 1M-context recipe. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Engram tables move to pinned host memory at TP2 and +# at c128, where resident tables leave too little KV. Graph capture covers +# 2x CONC (at least 128 sequences) x (1 + 5 drafts), up to 2048 tokens. The +# prefill chunk shrinks at TP2 from c64 and at c128 to buy KV room, and where +# it drops below 1024 x 6 slots the scheduler is capped at the captured shape. +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":false}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '1'}}} +override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":false}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '2'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":false}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '4'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":false}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '8'}}} +override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":false}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '16'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":false}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '32'}}} +override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":false}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '64'}}} +override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 2048, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '128'}}} +override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '1'}}} +override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '2'}}} +override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '4'}}} +override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '8'}}} +override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '16'}}} +override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '32'}}} +override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '64'}}} +override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 2048, max-num-batched-tokens: 4096, max-num-seqs: 256}}}, benchmark: {env: {CONC: '128'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index febbc02db8..e55023229f 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1359,8 +1359,8 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: # TP=2 c64, TP=4 c128 and TP=2 c128. Every point is measured here rather # than combined from the earlier run: the image move leaves concurrency # 1-32 measured only on the superseded nightly-eed1f3d0 pin. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } # Speculative decoding on an agentic scenario must run with simulated # synthetic acceptance at the committed golden AL for this model, thinking mode From 286801b87809767adda89c38150093e64097f663 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:34:23 -0500 Subject: [PATCH 067/105] feat(agentx): port the MiniMax-M3 MI355X vLLM AgentX config to srt-slurm --- .../vllm/mi355x-fp4-mtp/agentic.yaml | 85 +++++++++++++++++++ configs/amd-master.yaml | 4 +- 2 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..8ec01c32cf --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,85 @@ +# MiniMax-M3 MXFP4 AgentX on MI355X with vLLM EAGLE3 (GQA draft). The KV cache +# is GPU-resident. +base: + schema: 2 + name: minimaxm3-fp4-mi355x-vllm-agentic + model: + path: hf:amd/MiniMax-M3-MXFP4 + container: vllm/vllm-openai-rocm:nightly-2a02f6efe319c885e3ccbcecde402e0028f9ec1e + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: amd/MiniMax-M3-MXFP4 + trust-remote-code: true + block-size: 128 + gpu-memory-utilization: 0.90 + enable-chunked-prefill: true + max-num-batched-tokens: 32768 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY", "cudagraph_capture_sizes": [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,22,24,26,28,30,32,34,36,48,64,72,80,88,96,104,112,120,128]}' + language-model-only: true + enable-prefix-caching: true + attention-backend: ROCM_AITER_UNIFIED_ATTN + moe-backend: aiter + kv-cache-dtype: fp8 + attention-config: '{"indexer_kv_dtype": "fp8"}' + tool-call-parser: minimax_m3 + reasoning-parser: minimax_m3 + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + stream-interval: 20 + # Three-token EAGLE3; throughput runs replace verification with the + # golden acceptance length. + speculative-config: '{"method": "eagle3", "model": "Inferact/MiniMax-M3-EAGLE3-GQA", "num_speculative_tokens": 3, "attention_backend": "ROCM_AITER_UNIFIED_ATTN"}' + env: + PYTHONNOUSERSITE: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + VLLM_ROCM_USE_AITER: '1' + VLLM_ROCM_USE_AITER_MOE: '1' + VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: '1' + VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT: '1' + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 + VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16: '0' + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB: '256' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: amd/MiniMax-M3-MXFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. Admission is 2x CONC. +override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 2}}}, benchmark: {env: {CONC: '1'}}} +override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 8}}}, benchmark: {env: {CONC: '4'}}} +override_tp4_c5: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 10}}}, benchmark: {env: {CONC: '5'}}} +override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} +override_tp4_c10: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 20}}}, benchmark: {env: {CONC: '10'}}} +override_tp4_c12: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 24}}}, benchmark: {env: {CONC: '12'}}} +override_tp4_c15: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 30}}}, benchmark: {env: {CONC: '15'}}} +override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 40}}}, benchmark: {env: {CONC: '20'}}} +override_tp4_c24: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 48}}}, benchmark: {env: {CONC: '24'}}} +override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 64}}}, benchmark: {env: {CONC: '32'}}} +override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-num-seqs: 2}}}, benchmark: {env: {CONC: '1'}}} +override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-num-seqs: 4}}}, benchmark: {env: {CONC: '2'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index e55023229f..b07cc5ddec 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1234,8 +1234,8 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.20 search-space: - - { tp: 4, kv-offloading: none, conc-list: [1, 4, 5, 8, 10, 12, 15, 20, 24, 32], spec-decoding: mtp } - - { tp: 2, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp } + - { tp: 4, kv-offloading: none, conc-list: [1, 4, 5, 8, 10, 12, 15, 20, 24, 32], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agentic.yaml } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via SGLang with MTP speculative # decoding. Two arms: From 68d9a24829d82f5d8b903a53d91706551415d355 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:34:56 -0500 Subject: [PATCH 068/105] feat(srt): accept vLLM decode context parallelism and non-drafting points in the single-node adapter Kimi-K3 B300 serves TP8 with decode-context-parallel-size 8, and stops drafting above conc 16 while its matrix labels every point mtp; such a variant declares SPEC_DECODING in its benchmark env. --- infx/srt_slurm/single_node.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/infx/srt_slurm/single_node.py b/infx/srt_slurm/single_node.py index fcf8fef7ee..c20803d7a4 100644 --- a/infx/srt_slurm/single_node.py +++ b/infx/srt_slurm/single_node.py @@ -90,7 +90,8 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N spec = spec_parameters(role, engine) if spec and spec["method"] not in {"eagle", "eagle3", "nextn", "mtp", "dspark"}: raise ValueError("Single-node SRT supports only native MTP, EAGLE3, DSpark or no speculation") - speculation = "mtp" if spec else "none" + # A point that stops drafting may keep its matrix label. + speculation = "mtp" if spec else workload.get("SPEC_DECODING", "none") agentic = environment["IS_AGENTIC"] == "1" expected = { "engine": (engine, SINGLE_NODE_ENGINES[environment["FRAMEWORK"]]), @@ -119,7 +120,9 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N if engine == "atom": # Native ATOM derives -tp from the aggregate worker's GPU allocation. expected["ATOM TP"] = (role["gpus"], int(environment["TP"])) - for name, value in {"PP_SIZE": "1", "DCP_SIZE": "1", "PCP_SIZE": "1"}.items(): + # vLLM shards decode KV across its tensor-parallel ranks. + dcp = str(args.get("decode-context-parallel-size", 1)) if engine == "vllm" else "1" + for name, value in {"PP_SIZE": "1", "DCP_SIZE": dcp, "PCP_SIZE": "1"}.items(): expected[name] = (environment[name], value) for name, (actual, wanted) in expected.items(): if actual != wanted: From 7152530f4b1e441bedeb6a9307b5716da329a09d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:34:56 -0500 Subject: [PATCH 069/105] feat(agentx): port the Kimi-K3 B300 vLLM AgentX config to srt-slurm --- .../configs/kimik3-b300-mooncake.sh | 37 +++++ .../kimik3/vllm/b300-fp4-mtp/agentic.yaml | 126 ++++++++++++++++++ configs/nvidia-master.yaml | 2 +- 3 files changed, 164 insertions(+), 1 deletion(-) create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh create mode 100644 benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh new file mode 100755 index 0000000000..5f31c693c4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Pin the worker's Mooncake client and point its store at one active RDMA rail. +set -euo pipefail +pip_install=(python3 -m pip install) +if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages; then + pip_install+=(--break-system-packages) +fi +"${pip_install[@]}" --quiet --no-cache-dir --no-deps --force-reinstall \ + mooncake-transfer-engine-cuda13==0.3.11.post1 +python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null + +# Rail-isolated nodes: two RNICs cannot reach each other even within a node, so +# every rank uses one rail. mlx5_0 is down on some nodes, and topology discovery +# then finds no HCA, so take the first active rail at runtime. +rail="" +for device in mlx5_0 mlx5_1 mlx5_2 mlx5_3 mlx5_4 mlx5_5 mlx5_8 mlx5_9 \ + mlx5_10 mlx5_11 mlx5_16 mlx5_17 mlx5_20 mlx5_21 mlx5_22 mlx5_23; do + if grep -q ACTIVE "/sys/class/infiniband/$device/ports/1/state" 2>/dev/null; then + rail="$device" + break + fi +done +if [[ -z "$rail" ]]; then + echo "Error: no active RDMA rail on $(hostname); Mooncake cannot initialise" >&2 + exit 1 +fi +config="${MOONCAKE_CONFIG_PATH:-/logs/mooncake_store_config.json}" +python3 - "$config" "$rail" <<'PY' +import json, sys +path, rail = sys.argv[1:] +with open(path) as handle: + config = json.load(handle) +config["device_name"] = rail +with open(path, "w") as handle: + json.dump(config, handle, indent=2) +PY +echo "Mooncake rail: $rail" diff --git a/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..d3a1a26731 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,126 @@ +# Kimi-K3 MXFP4 AgentX on B300 with vLLM: TP8 x DCP8 with TokenspeedMLA and +# Mooncake as the external KV tier. DSpark drafts seven tokens up to CONC 8, +# three at CONC 16 and none above. +base: + schema: 2 + name: kimik3-fp4-b300-vllm-agentic + model: + path: hf:moonshotai/Kimi-K3 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # ~1.5 TB of MXFP4 shards may take the full VLLM_ENGINE_READY_TIMEOUT_S. + health_check: + interval_seconds: 10 + max_attempts: 360 + # Embedded Mooncake: each TP rank contributes TOTAL_CPU_DRAM_GB / 8 GB. The + # setup script pins the client to the master's version and fills in the + # node's active RDMA rail. + setup_script: kimik3-b300-mooncake.sh + services: + - name: mooncake-master + type: mooncake-master + preamble: >- + python3 -m pip install --break-system-packages --quiet --no-cache-dir --no-deps + --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 + args: ["--eviction_high_watermark_ratio=0.95", "--eviction_ratio=0.10"] + options: + store_config: + mode: embedded + metadata_server: P2PHANDSHAKE + global_segment_size: 281GB + local_buffer_size: 4GB + protocol: rdma + device_name: "" + enable_offload: false + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: moonshotai/Kimi-K3 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + load-format: fastsafetensors + moe-backend: auto + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: fp8 + stream-interval: 10 + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + disable-uvicorn-access-log: true + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + # These default to auto; name them so the measured DCP a2a path runs. + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + # Identical prefixes must hash to identical block keys run-to-run. + PYTHONHASHSEED: '42' + # Outlast AIPerf's pooled-connection idle gaps. + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900' + MC_GID_INDEX: '3' + MC_STORE_MEMCPY: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + MC_SLICE_SIZE: '1048576' + MC_WORKERS_PER_CTX: '4' + WITH_NVIDIA_PEERMEM: '0' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: moonshotai/Kimi-K3 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2249' + +# One variant per point. Admission is 2x CONC; CONC 56 and 70 keep more memory +# headroom. Graphs capture (1 + drafts) x 1..min(2x CONC, 128) tokens, then the +# larger powers of two to 8192. Throughput runs switch DSpark to synthetic +# rejection at the golden acceptance length; points above CONC 16 do not draft +# and keep the matrix's mtp label. +override_c1: {roles: {agg: {args: {max-num-seqs: 2, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,64,128,256,512,1024,2048,4096,8192]}', speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'}}}, benchmark: {env: {CONC: '1'}}} +override_c2: {roles: {agg: {args: {max-num-seqs: 4, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,64,128,256,512,1024,2048,4096,8192]}', speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'}}}, benchmark: {env: {CONC: '2'}}} +override_c4: {roles: {agg: {args: {max-num-seqs: 8, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,128,256,512,1024,2048,4096,8192]}', speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'}}}, benchmark: {env: {CONC: '4'}}} +override_c8: {roles: {agg: {args: {max-num-seqs: 16, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,256,512,1024,2048,4096,8192]}', speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'}}}, benchmark: {env: {CONC: '8'}}} +override_c16: {roles: {agg: {args: {max-num-seqs: 32, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,256,512,1024,2048,4096,8192]}', speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'}}}, benchmark: {env: {CONC: '16'}}} +override_c24: {roles: {agg: {args: {max-num-seqs: 48, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,64,128,256,512,1024,2048,4096,8192]}'}}}, benchmark: {env: {CONC: '24', SPEC_DECODING: mtp}}} +override_c32: {roles: {agg: {args: {max-num-seqs: 64, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,128,256,512,1024,2048,4096,8192]}'}}}, benchmark: {env: {CONC: '32', SPEC_DECODING: mtp}}} +override_c40: {roles: {agg: {args: {max-num-seqs: 80, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,128,256,512,1024,2048,4096,8192]}'}}}, benchmark: {env: {CONC: '40', SPEC_DECODING: mtp}}} +override_c48: {roles: {agg: {args: {max-num-seqs: 96, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,128,256,512,1024,2048,4096,8192]}'}}}, benchmark: {env: {CONC: '48', SPEC_DECODING: mtp}}} +override_c56: {roles: {agg: {args: {max-num-seqs: 112, gpu-memory-utilization: 0.9, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,128,256,512,1024,2048,4096,8192]}'}}}, benchmark: {env: {CONC: '56', SPEC_DECODING: mtp}}} +override_c70: {roles: {agg: {args: {max-num-seqs: 140, gpu-memory-utilization: 0.9, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,256,512,1024,2048,4096,8192]}'}}}, benchmark: {env: {CONC: '70', SPEC_DECODING: mtp}}} diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index e6b693c869..ea86890a8d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1529,7 +1529,7 @@ kimik3-fp4-b300-vllm-agentic-dspark: # One entry for every arm: the recipe drafts at DSpark 7 up to conc 8, # DSpark 3 at conc 16, and not at all above. Keep the concurrencies # disjoint across arms so exp-names stay unique. - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 70] } + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 70], srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml } dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 From e7bb22b7041d77c6e9309117ed5c9ededb9f487a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:36:41 -0500 Subject: [PATCH 070/105] feat(agentx): port the Kimi-K3 MI355X vLLM AgentX DCP1 points to srt-slurm The DCP8 no-draft points keep the legacy script: the single-node adapter requires DCP_SIZE=1 and speculation matching the matrix label. --- .../kimik3/vllm/mi355x-fp4-mtp/agentic.yaml | 90 +++++++++++++++++++ configs/amd-master.yaml | 4 +- 2 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml diff --git a/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..10982888eb --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,90 @@ +# Kimi-K3 MXFP4 AgentX on MI355X with vLLM DSpark +# (https://recipes.vllm.ai/moonshotai/Kimi-K3). TP8 only: the 1.56 TB checkpoint +# is ~195 GB per GPU. The KV cache is GPU-resident through c4 and backed by +# vLLM's SimpleCPUOffloadConnector from c8. The DCP8 arm (c44-c70) runs without +# a draft model and stays on the legacy script. +base: + schema: 2 + name: kimik3-fp4-mi355x-vllm-agentic + model: + path: hf:moonshotai/Kimi-K3 + container: vllm/vllm-openai-rocm:nightly-rocm100-af1c01499b289be555c475669ba50a88e96d846e + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # Weights load for up to VLLM_ENGINE_READY_TIMEOUT_S. + health_check: + interval_seconds: 10 + max_attempts: 720 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: moonshotai/Kimi-K3 + trust-remote-code: true + moe-backend: auto + tensor-parallel-size: 8 + load-format: fastsafetensors + gpu-memory-utilization: 0.9 + language-model-only: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + max-model-len: 1048576 + enable-prefix-caching: true + kv-cache-dtype: fp8 + attention-config: '{"mla_prefill_backend":"ROCM_AITER_FA"}' + env: + # Upstream AMD recipe environment. + VLLM_ROCM_AITER_MLA_ASM_PADDING: asm + VLLM_ROCM_USE_AITER: '1' + SAFETENSORS_FAST_GPU: '1' + VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4: '1' + AITER_SITUV2_A8W4: '1' + AITER_BF16_FP8_MOE_BOUND: '0' + AITER_QUICK_REDUCE_QUANTIZATION: INT4 + # The MI355X nodes report MEC firmware 38, below the 177 that fixes the + # RCCL memory reclaim issue. + HSA_NO_SCRATCH_RECLAIM: '1' + # 2.8 TB of weights off a shared mount takes far longer than the default. + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONNOUSERSITE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1200' + VLLM_USE_DIRECT_DCP_A2A: '0' + VLLM_USE_DIRECT_DCP_Q_GATHER: '0' + VLLM_USE_DIRECT_DCP_KV_GATHER: '0' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: moonshotai/Kimi-K3 + # Long agentic turns against a 1M context are prefill-bound on the server. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. The DSpark draft length descends with concurrency +# (7, 5, 4, 4, 3, 3); throughput runs replace block rejection with the golden +# acceptance length for that draft length. Admission is 2x CONC and graph +# capture covers every size up to 2x CONC x (1 + drafts). Breakable +# FULL_AND_PIECEWISE graphs cost KV pool, so only c1 and c4 use them. The +# offload points split the 1799 GB DRAM budget evenly across the eight ranks; +# identical prefixes must hash to identical block keys on every rank. +override_tp8_c1: {roles: {agg: {args: {max-num-seqs: 2, max-num-batched-tokens: 16384, compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]}', speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'}, env: {VLLM_USE_BREAKABLE_CUDAGRAPH: '1'}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: 'none'}}} +override_tp8_c4: {roles: {agg: {args: {max-num-seqs: 8, max-num-batched-tokens: 8192, compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":48,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48]}', speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":5,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'}, env: {VLLM_USE_BREAKABLE_CUDAGRAPH: '1'}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: 'none'}}} +override_tp8_c8_simple: {roles: {agg: {args: {max-num-seqs: 16, max-num-batched-tokens: 8192, compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":80,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80]}', speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}'}, env: {VLLM_USE_BREAKABLE_CUDAGRAPH: '0', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '1799'}}} +override_tp8_c10_simple: {roles: {agg: {args: {max-num-seqs: 20, max-num-batched-tokens: 8192, compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":100,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100]}', speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}'}, env: {VLLM_USE_BREAKABLE_CUDAGRAPH: '0', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '1799'}}} +override_tp8_c12_simple: {roles: {agg: {args: {max-num-seqs: 24, max-num-batched-tokens: 8192, compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":96,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96]}', speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}'}, env: {VLLM_USE_BREAKABLE_CUDAGRAPH: '0', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '1799'}}} +override_tp8_c14_simple: {roles: {agg: {args: {max-num-seqs: 28, max-num-batched-tokens: 8192, compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":112,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112]}', speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}'}, env: {VLLM_USE_BREAKABLE_CUDAGRAPH: '0', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '14', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '1799'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b07cc5ddec..d27ab6b94b 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -664,8 +664,8 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.60 search-space: - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1, 4] , spec-decoding: mtp} - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 10, 12, 14], spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1, 4] , spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 10, 12, 14], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [44, 48, 70], spec-decoding: mtp } # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark From 2e668e69010096d7218ed60dadb3bc0d7f919d74 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:37:19 -0500 Subject: [PATCH 071/105] fix(matrix): read node counts from named srt-slurm override variants recipe_node_count returned None for any CONFIG_FILE carrying a selector, so rows pointing at `file.yaml:override_` fell back to the master topology estimate. Resolve `base` and `override_` the way srtctl does (deep merge, null deletes, top-level schema carried into the variant) so the recipe allocation stays authoritative. Zip groups and non-schema-2 variant files keep the estimate. --- infx/matrix/generate.py | 31 ++++++++++++++++--- .../test_generate_sweep_configs.py | 24 ++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/infx/matrix/generate.py b/infx/matrix/generate.py index b4fc61deb1..28d0f2a1f3 100644 --- a/infx/matrix/generate.py +++ b/infx/matrix/generate.py @@ -223,6 +223,19 @@ def _worker_node_override(worker: dict, setting_name: str) -> int | None: return values[0] +def _merge_recipe(base: dict, override: dict) -> dict: + """Deep-merge an srt-slurm override variant over its base, as srtctl does.""" + merged = dict(base) + for key, value in override.items(): + if value is None: + merged.pop(key, None) + elif isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = _merge_recipe(merged[key], value) + else: + merged[key] = value + return merged + + def recipe_node_count(prefill: dict, decode: dict) -> int | None: """Read the authoritative node count from a checked-in srt-slurm recipe.""" config_files = { @@ -236,7 +249,7 @@ def recipe_node_count(prefill: dict, decode: dict) -> int | None: if len(config_files) != 1: raise ValueError(f"Conflicting CONFIG_FILE settings: {sorted(config_files)}") - config_file = config_files.pop() + config_file, _, selector = config_files.pop().partition(":") repo_root = repository_root() recipe_root = repo_root / "benchmarks" / "multi_node" / "srt-slurm-recipes" if config_file.startswith("benchmarks/multi_node/srt-slurm-recipes/"): @@ -249,12 +262,20 @@ def recipe_node_count(prefill: dict, decode: dict) -> int | None: return None recipe = yaml.safe_load(recipe_path.read_text()) + if "base" in recipe: + # srtctl merges a named variant over base (null deletes a key) and + # carries a top-level schema into it. Zip groups and non-schema-2 + # variant files have no authoritative count here; the selected master + # topology supplies the estimate. + if not (selector == "base" or (selector.startswith("override_") and selector in recipe)): + return None + schema = recipe.get("schema") + recipe = _merge_recipe(recipe["base"], recipe.get(selector) or {}) + recipe.setdefault("schema", schema) + if recipe.get("schema") != 2: + return None if recipe.get("schema") != 2: raise ValueError(f"srt-slurm recipes must declare schema: 2: {recipe_path}") - if "base" in recipe: - # A file with several override variants has no single authoritative - # node count. The selected master topology supplies the estimate. - return None roles = recipe.get("roles") if roles: # Schema 2 groups node allocations by role. A colocated decode role diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index a09db34998..732827c59b 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -121,6 +121,30 @@ def test_multinode_node_count_reads_schema_two_roles(tmp_path, monkeypatch, role assert generate.recipe_node_count(prefill, {}) == expected +@pytest.mark.parametrize("selector, expected", [ + ("base", 3), + ("override_wide", 5), + ("override_colocated", 2), + ("zip_override_sweep[0]", None), +]) +def test_recipe_node_count_resolves_override_selectors(tmp_path, monkeypatch, selector, expected): + recipe = tmp_path / "benchmarks/multi_node/srt-slurm-recipes/variants.yaml" + recipe.parent.mkdir(parents=True) + recipe.write_text(yaml.safe_dump({ + "schema": 2, + "base": {"roles": {"prefill": {"nodes": 1}, "decode": {"nodes": 2}}}, + "override_wide": {"roles": {"decode": {"nodes": 4}}}, + "override_colocated": {"roles": {"prefill": {"nodes": 2}, "decode": {"nodes": "colocate"}}}, + "zip_override_sweep": {"roles": {"decode": {"nodes": [1, 2]}}}, + })) + import infx.matrix.generate as generate + import infx.config + (tmp_path / "configs").mkdir() + monkeypatch.setattr(infx.config, "__file__", str(tmp_path / "infx/config.py")) + prefill = {"additional-settings": [f"CONFIG_FILE=recipes/variants.yaml:{selector}"]} + assert generate.recipe_node_count(prefill, {}) == expected + + def test_multinode_node_count_uses_role_gpu_footprints(sample_runner_config): prefill = {"num-worker": 3, "tp": 2, "pp": 1, "pcp-size": 1} decode = {"num-worker": 2, "tp": 8, "pp": 1, "pcp-size": 1} From e236d7f33ec4e90f317faffc9f0dfb8c9d3515b2 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:37:31 -0500 Subject: [PATCH 072/105] refactor(agentx): consolidate DSV4 multi-node AgentX recipes into override variants Each per-configuration recipe becomes an `override_` block over a shared `base` in one `*-variants.yaml` per master-config entry, and the master entries select it with `CONFIG_FILE=...:override_`. Every selected variant resolves, through the pinned srtctl, to exactly the recipe it replaces, including its original `name`. Power recipes with top-level telemetry stay standalone because launchers detect them as text. --- .../agentx/agg-b200-tp8-c1-mtp-hicache.yaml | 136 ------ .../agentx/agg-b200-tp8-c4-mtp-hicache.yaml | 136 ------ .../agentx/agg-b200-tp8-c8-mtp-hicache.yaml | 136 ------ .../sglang/b200-fp4/agentx/agg-variants.yaml | 151 +++++++ ...200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml | 234 ---------- ...b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml | 234 ---------- ...200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml | 239 ---------- .../b200-fp4/agentx/disagg-variants.yaml | 263 +++++++++++ .../sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml | 130 ------ .../sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml | 130 ------ .../sglang/gb300-fp4/agentx/agg-variants.yaml | 146 ++++++ ...gg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml | 234 ---------- ...gg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml | 236 ---------- ...g-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml | 234 ---------- ...g-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml | 239 ---------- .../gb300-fp4/agentx/disagg-variants.yaml | 313 +++++++++++++ .../disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml | 195 --------- .../disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml | 195 --------- .../disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml | 195 --------- .../disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml | 196 --------- .../disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml | 202 --------- .../disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml | 218 --------- .../gb300-fp4/agentx/disagg-variants.yaml | 399 +++++++++++++++++ .../vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml | 112 ----- .../gb200-fp4/agentx/agg-mtp-variants.yaml | 135 ++++++ .../gb200-fp4/agentx/agg-mtp3-variants.yaml | 176 ++++++++ .../gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml | 160 ------- .../gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml | 159 ------- .../vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml | 122 ------ .../disagg-1p1d-dep8-dep8-c128-mtp3.yaml | 221 ---------- .../disagg-1p1d-dep8-dep8-c256-mtp3.yaml | 221 ---------- .../agentx/disagg-1p1d-dep8-dep8-mtp.yaml | 133 ------ .../disagg-2p1d-dep8-dep12-c576-mtp3.yaml | 223 ---------- .../disagg-2p1d-dep8-dep16-c512-mtp3.yaml | 223 ---------- .../agentx/disagg-2p1d-dep8-dep8-mtp.yaml | 130 ------ .../gb200-fp4/agentx/disagg-mtp-variants.yaml | 175 ++++++++ .../agentx/disagg-mtp3-variants.yaml | 303 +++++++++++++ .../vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml | 159 ------- .../vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml | 160 ------- .../vllm/gb300-fp4/agentx/agg-variants.yaml | 182 ++++++++ .../disagg-1p1d-dep4-dep16-c128-mtp.yaml | 228 ---------- .../disagg-1p1d-dep4-dep16-c256-mtp.yaml | 228 ---------- .../disagg-1p1d-dep4-dep8-c256-mtp.yaml | 221 ---------- .../disagg-1p1d-dep8-dep16-c512-mtp.yaml | 228 ---------- .../agentx/disagg-1p6d-dep4-tp4.yaml | 179 -------- .../disagg-2p1d-dep8-dep12-c1152-mtp.yaml | 223 ---------- .../disagg-2p1d-dep8-dep16-c1024-mtp.yaml | 223 ---------- .../disagg-4p1d-dep4-dep8-24-c4096.yaml | 188 -------- .../gb300-fp4/agentx/disagg-mtp-variants.yaml | 414 ++++++++++++++++++ .../gb300-fp4/agentx/disagg-variants.yaml | 222 ++++++++++ configs/nvidia-master.yaml | 82 ++-- 51 files changed, 2920 insertions(+), 7301 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c1-mtp-hicache.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c4-mtp-hicache.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c8-mtp-hicache.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c1-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c1-mtp-hicache.yaml deleted file mode 100644 index c24d199b7c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c1-mtp-hicache.yaml +++ /dev/null @@ -1,136 +0,0 @@ -schema: 2 -name: "agg-b200-tp8-c1-mtp-hicache" - -# B200 AgentX aggregate topology: one TP8 worker uses one eight-GPU node -# and serves both prefill and decode with bundled DSpark and HiCache. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" - frameworks: - dynamo: "1.5.0.dev20260914" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "b200" - gpus_per_node: 8 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 8 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: high - PIP_BREAK_SYSTEM_PACKAGES: "1" - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" - enable-metrics: true - enable-cache-report: true - trust-remote-code: true - weight-loader-prefetch-checkpoints: true - stream-interval: 10 - watchdog-timeout: 1000000 - mem-fraction-static: 0.90 - page-size: 256 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - moe-runner-backend: "flashinfer_mxfp4" - enable-deepseek-v4-fp4-indexer: true - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 2 - cuda-graph-max-bs-decode: 2 - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 8 - ep-size: 1 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-ratio: 2.75 - hicache-write-policy: write_through - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - -sbatch_directives: - mem: "0" - # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; - # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. - cpus-per-task: "192" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "false" - TP: "8" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c4-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c4-mtp-hicache.yaml deleted file mode 100644 index 06b0f96500..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c4-mtp-hicache.yaml +++ /dev/null @@ -1,136 +0,0 @@ -schema: 2 -name: "agg-b200-tp8-c4-mtp-hicache" - -# B200 AgentX aggregate topology: one TP8 worker uses one eight-GPU node -# and serves both prefill and decode with bundled DSpark and HiCache. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" - frameworks: - dynamo: "1.5.0.dev20260914" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "b200" - gpus_per_node: 8 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 8 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: high - PIP_BREAK_SYSTEM_PACKAGES: "1" - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" - enable-metrics: true - enable-cache-report: true - trust-remote-code: true - weight-loader-prefetch-checkpoints: true - stream-interval: 10 - watchdog-timeout: 1000000 - mem-fraction-static: 0.90 - page-size: 256 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - moe-runner-backend: "flashinfer_mxfp4" - enable-deepseek-v4-fp4-indexer: true - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 8 - cuda-graph-max-bs-decode: 8 - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 8 - ep-size: 1 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-ratio: 2.75 - hicache-write-policy: write_through - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - -sbatch_directives: - mem: "0" - # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; - # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. - cpus-per-task: "192" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "false" - TP: "8" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c8-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c8-mtp-hicache.yaml deleted file mode 100644 index 403e23ca15..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c8-mtp-hicache.yaml +++ /dev/null @@ -1,136 +0,0 @@ -schema: 2 -name: "agg-b200-tp8-c8-mtp-hicache" - -# B200 AgentX aggregate topology: one TP8 worker uses one eight-GPU node -# and serves both prefill and decode with bundled DSpark and HiCache. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" - frameworks: - dynamo: "1.5.0.dev20260914" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "b200" - gpus_per_node: 8 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 8 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: high - PIP_BREAK_SYSTEM_PACKAGES: "1" - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" - enable-metrics: true - enable-cache-report: true - trust-remote-code: true - weight-loader-prefetch-checkpoints: true - stream-interval: 10 - watchdog-timeout: 1000000 - mem-fraction-static: 0.90 - page-size: 256 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - moe-runner-backend: "flashinfer_mxfp4" - enable-deepseek-v4-fp4-indexer: true - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 8 - ep-size: 1 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-ratio: 2.75 - hicache-write-policy: write_through - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - -sbatch_directives: - mem: "0" - # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; - # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. - cpus-per-task: "192" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "false" - TP: "8" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..3e302a5ac7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml @@ -0,0 +1,151 @@ +# AgentX dsv4 sglang b200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + # B200 AgentX aggregate topology: one TP8 worker uses one eight-GPU node + # and serves both prefill and decode with bundled DSpark and HiCache. + model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" + frameworks: + dynamo: "1.5.0.dev20260914" + dynamo: + install: true + source: + wheel: "1.5.0.dev20260914" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: "b200" + gpus_per_node: 8 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" + enable-metrics: true + enable-cache-report: true + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.90 + page-size: 256 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + enable-deepseek-v4-fp4-indexer: true + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 8 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + enable-hierarchical-cache: true + hicache-ratio: 2.75 + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + sbatch_directives: + mem: "0" + # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; + # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. + cpus-per-task: "192" + srun_options: + mem: "0" + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "8" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + +override_c1: + name: "agg-b200-tp8-c1-mtp-hicache" + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + +override_c4: + name: "agg-b200-tp8-c4-mtp-hicache" + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + +override_c8: + name: "agg-b200-tp8-c8-mtp-hicache" + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml deleted file mode 100644 index 10a55422a4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml +++ /dev/null @@ -1,234 +0,0 @@ -schema: 2 -name: "disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload" - -# B200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 -# (1P x DEP8 / 1D x DEP8, bundled DSpark + HiCache KV offload), tuned for concurrency 128. -# Each DEP8 worker occupies one eight-GPU B200 node. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" - frameworks: - dynamo: "1.5.0.dev20260914" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: b200 - gpus_per_node: 8 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 8 - env: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - # P/D workers are on separate HGX B200 nodes, so use Mooncake's standard - # RDMA-registered CUDA buffers rather than an NVL72 custom memory pool. - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - # Use the eight external HCAs. mlx5_6..9 are on a separate low-LID - # fabric, while mlx5_bond_0 caused cross-fabric QP RTR timeouts. - disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - # Favor the full-attention pool; measured SWA utilization remained low. - swa-full-tokens-ratio: 0.01 - max-running-requests: 256 - cuda-graph-max-bs-decode: 256 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 8 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - - decode: - nodes: 1 - workers: 1 - gpus: 8 - - env: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_RAGGED_VERIFY_MODE: "static" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 256 - cuda-graph-max-bs-decode: 256 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; - # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. - cpus-per-task: "192" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml deleted file mode 100644 index bfe784c531..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml +++ /dev/null @@ -1,234 +0,0 @@ -schema: 2 -name: "disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload" - -# B200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 -# (1P x DEP8 / 1D x DEP8, bundled DSpark + HiCache KV offload), tuned for concurrency 64. -# Each DEP8 worker occupies one eight-GPU B200 node. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" - frameworks: - dynamo: "1.5.0.dev20260914" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: b200 - gpus_per_node: 8 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 8 - env: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - # P/D workers are on separate HGX B200 nodes, so use Mooncake's standard - # RDMA-registered CUDA buffers rather than an NVL72 custom memory pool. - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - # Use the eight external HCAs. mlx5_6..9 are on a separate low-LID - # fabric, while mlx5_bond_0 caused cross-fabric QP RTR timeouts. - disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - # Favor the full-attention pool; measured SWA utilization remained low. - swa-full-tokens-ratio: 0.01 - max-running-requests: 256 - cuda-graph-max-bs-decode: 256 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 8 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - - decode: - nodes: 1 - workers: 1 - gpus: 8 - - env: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_RAGGED_VERIFY_MODE: "static" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 128 - cuda-graph-max-bs-decode: 256 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; - # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. - cpus-per-task: "192" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml deleted file mode 100644 index ea011fe829..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml +++ /dev/null @@ -1,239 +0,0 @@ -schema: 2 -name: "disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload" - -# B200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 -# (2P x DEP8 / 1D x DEP8, bundled DSpark + HiCache KV offload), tuned for concurrency 256. -# Each DEP8 worker occupies one eight-GPU B200 node. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" - frameworks: - dynamo: "1.5.0.dev20260914" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: b200 - gpus_per_node: 8 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - prefill: - nodes: 2 - workers: 2 - gpus: 8 - env: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - # P/D workers are on separate HGX B200 nodes, so use Mooncake's standard - # RDMA-registered CUDA buffers rather than an NVL72 custom memory pool. - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_DSV4_MHC_PREWARM: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - # Use the eight external HCAs. mlx5_6..9 are on a separate low-LID - # fabric, while mlx5_bond_0 caused cross-fabric QP RTR timeouts. - disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 256 - cuda-graph-max-bs-decode: 256 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 8 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - - decode: - nodes: 1 - workers: 1 - gpus: 8 - - - env: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_RAGGED_VERIFY_MODE: "static" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_DSV4_MHC_PREWARM: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 - disaggregation-mode: decode - load-balance-method: total_tokens - # Leave enough decode activation headroom while expanding the KV pools - # for transient DP-rank imbalance at concurrency 256. - mem-fraction-static: 0.91 - page-size: 256 - # Favor the full-attention pool while retaining enough SWA capacity for - # the busiest decode rank. - swa-full-tokens-ratio: 0.005 - max-running-requests: 512 - cuda-graph-max-bs-decode: 256 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; - # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. - cpus-per-task: "192" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..9b729ab3fd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,263 @@ +# AgentX dsv4 sglang b200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" + frameworks: + dynamo: "1.5.0.dev20260914" + dynamo: + install: true + source: + wheel: "1.5.0.dev20260914" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: b200 + gpus_per_node: 8 + services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + # P/D workers are on separate HGX B200 nodes, so use Mooncake's standard + # RDMA-registered CUDA buffers rather than an NVL72 custom memory pool. + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + # Use the eight external HCAs. mlx5_6..9 are on a separate low-LID + # fabric, while mlx5_bond_0 caused cross-fabric QP RTR timeouts. + disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.85 + page-size: 256 + # Favor the full-attention pool; measured SWA utilization remained low. + swa-full-tokens-ratio: 0.01 + max-running-requests: 256 + cuda-graph-max-bs-decode: 256 + chunked-prefill-size: 65536 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 8 + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_RAGGED_VERIFY_MODE: "static" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + page-size: 256 + swa-full-tokens-ratio: 0.02 + cuda-graph-max-bs-decode: 256 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + sbatch_directives: + mem: "0" + # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; + # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. + cpus-per-task: "192" + srun_options: + mem: "0" + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + +# B200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 +# (1P x DEP8 / 1D x DEP8, bundled DSpark + HiCache KV offload), tuned for concurrency 64. +# Each DEP8 worker occupies one eight-GPU B200 node. +override_1p1d_c64: + name: "disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload" + roles: + decode: + args: + max-running-requests: 128 + +# B200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 +# (1P x DEP8 / 1D x DEP8, bundled DSpark + HiCache KV offload), tuned for concurrency 128. +# Each DEP8 worker occupies one eight-GPU B200 node. +override_1p1d_c128: + name: "disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload" + roles: + decode: + args: + max-running-requests: 256 + +# B200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 +# (2P x DEP8 / 1D x DEP8, bundled DSpark + HiCache KV offload), tuned for concurrency 256. +# Each DEP8 worker occupies one eight-GPU B200 node. +override_2p1d_c256: + name: "disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload" + roles: + prefill: + nodes: 2 + workers: 2 + args: + swa-full-tokens-ratio: 0.02 + decode: + env: + SGLANG_DSV4_MHC_PREWARM: '1' + args: + # Leave enough decode activation headroom while expanding the KV pools + # for transient DP-rank imbalance at concurrency 256. + mem-fraction-static: 0.91 + # Favor the full-attention pool while retaining enough SWA capacity for + # the busiest decode rank. + swa-full-tokens-ratio: 0.005 + max-running-requests: 512 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml deleted file mode 100644 index f270e4ffc8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml +++ /dev/null @@ -1,130 +0,0 @@ -schema: 2 -name: "agg-gb300-tp4-mtp-lowlatency" - -# Low-latency AgentX aggregate topology: one TP4 worker occupies one -# four-GPU GB300 node and serves both prefill and decode with DSpark K=6. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260910" - -slurm: - time_limit: "4:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: high - PIP_BREAK_SYSTEM_PACKAGES: "1" - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" - enable-metrics: true - enable-cache-report: true - trust-remote-code: true - weight-loader-prefetch-checkpoints: true - stream-interval: 10 - watchdog-timeout: 1000000 - mem-fraction-static: 0.94 - page-size: 256 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - moe-runner-backend: "flashinfer_mxfp4" - enable-deepseek-v4-fp4-indexer: true - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 32 - cuda-graph-max-bs-decode: 32 - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 4 - ep-size: 1 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "false" - TP: "4" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml deleted file mode 100644 index c633f7506f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml +++ /dev/null @@ -1,130 +0,0 @@ -schema: 2 -name: "agg-gb300-tp8-mtp-lowlatency" - -# Low-latency AgentX aggregate topology: one TP8 worker spans two -# four-GPU GB300 nodes and serves both prefill and decode with DSpark K=6. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260910" - -slurm: - time_limit: "4:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: high - PIP_BREAK_SYSTEM_PACKAGES: "1" - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" - enable-metrics: true - enable-cache-report: true - trust-remote-code: true - weight-loader-prefetch-checkpoints: true - stream-interval: 10 - watchdog-timeout: 1000000 - mem-fraction-static: 0.94 - page-size: 256 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - moe-runner-backend: "flashinfer_mxfp4" - enable-deepseek-v4-fp4-indexer: true - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 4 - cuda-graph-max-bs-decode: 4 - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 8 - ep-size: 1 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "false" - TP: "8" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..2ef179373b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -0,0 +1,146 @@ +# AgentX dsv4 sglang gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9" + dynamo: + install: true + source: + wheel: "1.5.0.dev20260910" + slurm: + time_limit: "4:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: "gb300" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + agg: + workers: 1 + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" + enable-metrics: true + enable-cache-report: true + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.94 + page-size: 256 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + enable-deepseek-v4-fp4-indexer: true + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + scheduler-recv-interval: 30 + dp-size: 1 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + sbatch_directives: + mem: "0" + cpus-per-task: "144" + srun_options: + mem: "0" + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + +# Low-latency AgentX aggregate topology: one TP4 worker occupies one +# four-GPU GB300 node and serves both prefill and decode with DSpark K=6. +override_tp4: + name: "agg-gb300-tp4-mtp-lowlatency" + roles: + agg: + nodes: 1 + gpus: 4 + args: + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + tp-size: 4 + benchmark: + env: + TP: "4" + +# Low-latency AgentX aggregate topology: one TP8 worker spans two +# four-GPU GB300 nodes and serves both prefill and decode with DSpark K=6. +override_tp8: + name: "agg-gb300-tp8-mtp-lowlatency" + roles: + agg: + nodes: 2 + gpus: 8 + args: + max-running-requests: 4 + cuda-graph-max-bs-decode: 4 + tp-size: 8 + benchmark: + env: + TP: "8" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml deleted file mode 100644 index 03bd3727d5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml +++ /dev/null @@ -1,234 +0,0 @@ -schema: 2 -name: "disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 480. -# -# Uses the flat single-variant srtctl schema the agentic CI flow expects; -# resources + backend (prefill/decode env + sglang_config) are normalized -# from the Pareto run. -# Concurrency is exported into srt_agentic.sh from the master-config conc-list. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260902" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 256 - cuda-graph-max-bs: 256 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 16 - dp-size: 16 - ep-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 3072 - cuda-graph-max-bs: 256 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml deleted file mode 100644 index 2ae7922cec..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml +++ /dev/null @@ -1,236 +0,0 @@ -schema: 2 -name: "disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 -# (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 960. -# -# DEP8-prefill variant aligned with the measured Pareto point: prefill uses -# tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. -# Concurrency is exported into srt_agentic.sh -# from the master-config conc-list. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260902" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - prefill: - nodes: 4 - workers: 2 - env: - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - OMP_NUM_THREADS: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_DSV4_MHC_PREWARM: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 256 - cuda-graph-max-bs: 256 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - nodes: 4 - workers: 1 - - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - OMP_NUM_THREADS: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_DSV4_MHC_PREWARM: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 16 - dp-size: 16 - ep-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 3072 - cuda-graph-max-bs: 256 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml deleted file mode 100644 index 95bb232ce5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml +++ /dev/null @@ -1,234 +0,0 @@ -schema: 2 -name: "disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1440. -# -# Uses the flat single-variant srtctl schema the agentic CI flow expects; -# resources + backend (prefill/decode env + sglang_config) are normalized -# from the Pareto run. -# Concurrency is exported into srt_agentic.sh from the master-config conc-list. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260902" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - prefill: - nodes: 6 - workers: 3 - gpus: 8 - env: - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 16 - dp-size: 16 - ep-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 3072 - cuda-graph-max-bs: 512 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml deleted file mode 100644 index f0de0d1381..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml +++ /dev/null @@ -1,239 +0,0 @@ -schema: 2 -name: "disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1920. -# -# Uses the flat single-variant srtctl schema the agentic CI flow expects; -# resources + backend (prefill/decode env + sglang_config) are normalized -# from the Pareto run. -# Concurrency is exported into srt_agentic.sh from the master-config conc-list. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260902" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - nginx_keepalive_timeout: "900s" - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - # AgentX warmup can legitimately keep the single wide decode worker busy - # for longer than Dynamo's 10-second TCP request-plane default. - DYN_TCP_REQUEST_TIMEOUT: "60" - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - prefill: - nodes: 8 - workers: 4 - gpus: 8 - env: - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 16 - dp-size: 16 - ep-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 3072 - cuda-graph-max-bs: 192 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..ad2af4be37 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,313 @@ +# AgentX dsv4 sglang gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" + dynamo: + install: true + source: + wheel: "1.5.0.dev20260902" + slurm: + time_limit: "8:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + prefill: + env: + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.85 + page-size: 256 + swa-full-tokens-ratio: 0.02 + chunked-prefill-size: 65536 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + decode: + nodes: 4 + workers: 1 + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 3072 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + sbatch_directives: + mem: "0" + cpus-per-task: "144" + srun_options: + mem: "0" + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 480. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into srt_agentic.sh from the master-config conc-list. +override_1p1d_c480: + name: "disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload" + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + args: + max-running-requests: 256 + cuda-graph-max-bs: 256 + decode: + gpus: 16 + args: + cuda-graph-max-bs: 256 + +# Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 +# (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 960. +# +# DEP8-prefill variant aligned with the measured Pareto point: prefill uses +# tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. +# Concurrency is exported into srt_agentic.sh +# from the master-config conc-list. +override_2p1d_c960: + name: "disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload" + roles: + prefill: + nodes: 4 + workers: 2 + env: + OMP_NUM_THREADS: '1' + args: + max-running-requests: 256 + cuda-graph-max-bs: 256 + decode: + env: + OMP_NUM_THREADS: '1' + SGLANG_DSV4_MHC_PREWARM: '1' + args: + cuda-graph-max-bs: 256 + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1440. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into srt_agentic.sh from the master-config conc-list. +override_3p1d_c1440: + name: "disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload" + roles: + prefill: + nodes: 6 + workers: 3 + gpus: 8 + args: + max-running-requests: 512 + cuda-graph-max-bs: 512 + decode: + gpus: 16 + args: + cuda-graph-max-bs: 512 + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1920. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into srt_agentic.sh from the master-config conc-list. +override_4p1d_c1920: + name: "disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload" + frontend: + nginx_keepalive_timeout: "900s" + env: + # AgentX warmup can legitimately keep the single wide decode worker busy + # for longer than Dynamo's 10-second TCP request-plane default. + DYN_TCP_REQUEST_TIMEOUT: "60" + roles: + prefill: + nodes: 8 + workers: 4 + gpus: 8 + env: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + args: + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + decode: + gpus: 16 + env: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + args: + cuda-graph-max-bs: 192 + benchmark: + env: + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml deleted file mode 100644 index beb34c24dd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml +++ /dev/null @@ -1,195 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 256 - max_num_tokens: 16384 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 8 - decode: - nodes: 8 - workers: 1 - gpus: 32 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.7 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 4 - max_num_tokens: 16 - max_seq_len: 996595 - moe_config: - backend: MEGAMOE_DEEPGEMM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 32 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '388' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: first_decode -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '388' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro - placement: - node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml deleted file mode 100644 index 27a8c25686..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml +++ /dev/null @@ -1,195 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 128 - max_num_tokens: 4096 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 4 - decode: - nodes: 8 - workers: 4 - gpus: 8 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.9 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 1 - max_num_tokens: 4 - max_seq_len: 996595 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 8 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '4' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: first_decode -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '4' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro - placement: - node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml deleted file mode 100644 index f6f914fd83..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml +++ /dev/null @@ -1,195 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 128 - max_num_tokens: 4096 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 4 - decode: - nodes: 6 - workers: 6 - gpus: 4 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.9 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 4 - max_num_tokens: 16 - max_seq_len: 996595 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '24' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: first_decode -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '24' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro - placement: - node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml deleted file mode 100644 index 1bf613d495..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml +++ /dev/null @@ -1,196 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 4 - workers: 2 - gpus: 8 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 256 - max_num_tokens: 16384 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 8 - decode: - nodes: 8 - workers: 1 - gpus: 32 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.7 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 8 - max_num_tokens: 32 - max_seq_len: 996595 - moe_config: - backend: MEGAMOE_DEEPGEMM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 32 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '736' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: first_decode -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '736' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro - placement: - node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml deleted file mode 100644 index 818f5da7cc..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml +++ /dev/null @@ -1,202 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 6 - workers: 3 - gpus: 8 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 256 - max_num_tokens: 16384 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 8 - decode: - nodes: 4 - workers: 1 - gpus: 16 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 12 - - 16 - - 20 - - 24 - - 28 - - 32 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.7 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 32 - max_num_tokens: 128 - max_seq_len: 996595 - moe_config: - backend: MEGAMOE_DEEPGEMM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 16 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '1152' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: first_decode -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '1152' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro - placement: - node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml deleted file mode 100644 index 7a5e0c4691..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml +++ /dev/null @@ -1,218 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 10 - workers: 5 - gpus: 8 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 256 - max_num_tokens: 16384 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 8 - decode: - nodes: 4 - workers: 1 - gpus: 16 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 12 - - 16 - - 20 - - 24 - - 28 - - 32 - - 36 - - 40 - - 44 - - 48 - - 52 - - 56 - - 60 - - 64 - - 68 - - 72 - - 76 - - 80 - - 84 - - 88 - - 92 - - 96 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.7 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 96 - max_num_tokens: 384 - max_seq_len: 996595 - moe_config: - backend: MEGAMOE_DEEPGEMM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 16 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '2626' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: first_decode -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '2626' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro - placement: - node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..d78570c07f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,399 @@ +# AgentX dsv4 trtllm gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + identity: + model: + repo: deepseek-ai/DeepSeek-V4-Pro + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + request_plane: tcp + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + engine: + type: trtllm + publish_events_and_metrics: false + roles: + prefill: + gpus: 8 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + CUDA_SCALE_LAUNCH_QUEUES: 4x + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 193273528320 + pool_ratio: + - 0.55 + - 0.22 + - 0.23 + tokens_per_block: 128 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + max_batch_size: 256 + max_num_tokens: 16384 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 8 + decode: + workers: 1 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.7 + host_cache_size: 0 + tokens_per_block: 128 + max_seq_len: 996595 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + frontend: + type: dynamo + enable_multiple_frontends: false + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + DURATION: '3600' + KV_OFFLOADING: dram + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: first_decode + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + DURATION: '3600' + KV_OFFLOADING: dram + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: DeepSeek-V4-Pro + placement: + node: last_decode + +override_1p4d_dep4_tep8_c4_b1: + name: dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + args: + max_batch_size: 128 + max_num_tokens: 4096 + moe_expert_parallel_size: 4 + tensor_parallel_size: 4 + decode: + nodes: 8 + workers: 4 + gpus: 8 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + free_gpu_memory_fraction: 0.9 + max_batch_size: 1 + max_num_tokens: 4 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + tensor_parallel_size: 8 + frontend: + env: + CONC: '4' + benchmark: + env: + CONC: '4' + +override_1p6d_dep4_tep4_c24_b4: + name: dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + args: + max_batch_size: 128 + max_num_tokens: 4096 + moe_expert_parallel_size: 4 + tensor_parallel_size: 4 + decode: + nodes: 6 + workers: 6 + gpus: 4 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + free_gpu_memory_fraction: 0.9 + max_batch_size: 4 + max_num_tokens: 16 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + tensor_parallel_size: 4 + frontend: + env: + CONC: '24' + benchmark: + env: + CONC: '24' + +override_1p1d_dep8_dep32_c388_b4: + name: dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp + roles: + prefill: + nodes: 2 + workers: 1 + decode: + nodes: 8 + gpus: 32 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + max_batch_size: 4 + max_num_tokens: 16 + moe_expert_parallel_size: 32 + tensor_parallel_size: 32 + frontend: + env: + CONC: '388' + benchmark: + env: + CONC: '388' + +override_2p1d_dep8_dep32_c736_b8: + name: dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp + roles: + prefill: + nodes: 4 + workers: 2 + decode: + nodes: 8 + gpus: 32 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + max_batch_size: 8 + max_num_tokens: 32 + moe_expert_parallel_size: 32 + tensor_parallel_size: 32 + frontend: + env: + CONC: '736' + benchmark: + env: + CONC: '736' + +override_3p1d_dep8_dep16_c1152_b32: + name: dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp + roles: + prefill: + nodes: 6 + workers: 3 + decode: + nodes: 4 + gpus: 16 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + max_batch_size: 32 + max_num_tokens: 128 + moe_expert_parallel_size: 16 + tensor_parallel_size: 16 + frontend: + env: + CONC: '1152' + benchmark: + env: + CONC: '1152' + +override_5p1d_dep8_dep16_c2626_b96: + name: dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp + roles: + prefill: + nodes: 10 + workers: 5 + decode: + nodes: 4 + gpus: 16 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + - 36 + - 40 + - 44 + - 48 + - 52 + - 56 + - 60 + - 64 + - 68 + - 72 + - 76 + - 80 + - 84 + - 88 + - 92 + - 96 + max_batch_size: 96 + max_num_tokens: 384 + moe_expert_parallel_size: 16 + tensor_parallel_size: 16 + frontend: + env: + CONC: '2626' + benchmark: + env: + CONC: '2626' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml deleted file mode 100644 index 49d15b0f32..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml +++ /dev/null @@ -1,112 +0,0 @@ -schema: 2 -name: dsv4-gb200-vllm-agentic-mtp-agg-dep8 - -model: - path: deepseek-v4-pro - container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - precision: fp4 - -identity: - model: {repo: deepseek-ai/DeepSeek-V4-Pro} - container: {image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106} - frameworks: {dynamo: "1.3.1"} - -dynamo: - install: true - - source: - pypi: "1.3.1" -setup_script: vllm-container-deps.sh -environment: {ETCD_LEASE_TTL: "7200"} -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: gb200 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: kv - router-reset-states: true - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 256 - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: /hf_hub_cache - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - args: - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 8192 - trust-remote-code: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - gpu-memory-utilization: 0.90 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - kv-cache-metrics: true - tokenizer-mode: deepseek_v4 - -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml new file mode 100644 index 0000000000..87fa054566 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml @@ -0,0 +1,135 @@ +# AgentX dsv4 vllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml:override_. + +schema: 2 + +base: + model: + path: deepseek-v4-pro + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + precision: fp4 + identity: + model: + repo: deepseek-ai/DeepSeek-V4-Pro + container: + image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + frameworks: + dynamo: "1.3.1" + dynamo: + install: true + source: + pypi: "1.3.1" + setup_script: vllm-container-deps.sh + environment: + ETCD_LEASE_TTL: "7200" + slurm: + time_limit: "8:00:00" + health_check: + max_attempts: 2160 + interval_seconds: 10 + resources: + gpu_type: gb200 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: kv + router-reset-states: true + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 256 + engine: + type: vllm + connector: + roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + pipeline-parallel-size: 1 + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-batched-tokens: 8192 + trust-remote-code: true + block-size: 256 + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + gpu-memory-utilization: 0.90 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + kv-cache-metrics: true + tokenizer-mode: deepseek_v4 + sbatch_directives: + cpus-per-task: "144" + mem: "0" + srun_options: + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + +override_tp8: + name: dsv4-gb200-vllm-agentic-mtp-agg-tp8 + roles: + agg: + args: + tensor-parallel-size: 8 + disable-custom-all-reduce: true + max-num-seqs: 128 + no-enable-flashinfer-autotune: true + +override_dep8: + name: dsv4-gb200-vllm-agentic-mtp-agg-dep8 + engine: + dp_launch_mode: per_node + roles: + agg: + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + max-num-seqs: 256 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml new file mode 100644 index 0000000000..bf9c7321ba --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml @@ -0,0 +1,176 @@ +# AgentX dsv4 vllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.2.1" + dynamo: + install: true + source: + wheel: "1.2.1" + environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + # The frontend shares Grace CPU capacity with the long TP8 cold start. + ETCD_LEASE_TTL: "7200" + setup_script: vllm-container-deps.sh + slurm: + time_limit: "8:00:00" + health_check: + max_attempts: 2160 + interval_seconds: 10 + resources: + gpu_type: "gb200" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + engine: + type: vllm + connector: + roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + gpu-memory-utilization: 0.94 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + sbatch_directives: + cpus-per-task: "144" + mem: "0" + srun_options: + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep aggregate workers in the multinode result schema so ingestion uses + # the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +# GB200 AgentX aggregate topology: one TP8 worker spans two four-GPU +# nodes and serves both prefill and decode at concurrency 4. Keep at least +# 16 sequence slots and otherwise size the scheduler at 4x concurrency. +override_c4: + name: "svf-vllm-agg-gb200-tp8-c4-mtp2-agentic" + roles: + agg: + env: + VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-agg-mtp2-{job_id}" + args: + max-num-seqs: 16 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' + +# GB200 AgentX aggregate topology: one TP8 worker spans two four-GPU +# nodes and serves both prefill and decode at concurrency 8. Size max-num-seqs at +# 4x concurrency and expand the MTP CUDA-graph envelope to match. +override_c8: + name: "svf-vllm-agg-gb200-tp8-c8-mtp2-agentic" + roles: + agg: + env: + VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-agg-tp8-c8-mtp2-{job_id}" + args: + max-num-seqs: 32 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' + benchmark: + env: + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml deleted file mode 100644 index c34d66a75c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml +++ /dev/null @@ -1,160 +0,0 @@ -schema: 2 -name: "svf-vllm-agg-gb200-tp8-c4-mtp2-agentic" - -# GB200 AgentX aggregate topology: one TP8 worker spans two four-GPU -# nodes and serves both prefill and decode at concurrency 4. Keep at least -# 16 sequence slots and otherwise size the scheduler at 4x concurrency. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - install: true - - source: - wheel: "1.2.1" -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - # The frontend shares Grace CPU capacity with the long TP8 cold start. - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -engine: - type: vllm - connector: -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-agg-mtp2-{job_id}" - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 16 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep aggregate workers in the multinode result schema so ingestion uses - # the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml deleted file mode 100644 index d9752e3eef..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml +++ /dev/null @@ -1,159 +0,0 @@ -schema: 2 -name: "svf-vllm-agg-gb200-tp8-c8-mtp2-agentic" - -# GB200 AgentX aggregate topology: one TP8 worker spans two four-GPU -# nodes and serves both prefill and decode at concurrency 8. Size max-num-seqs at -# 4x concurrency and expand the MTP CUDA-graph envelope to match. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - install: true - - source: - wheel: "1.2.1" -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -engine: - type: vllm - connector: -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-agg-tp8-c8-mtp2-{job_id}" - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml deleted file mode 100644 index 2fa022b714..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml +++ /dev/null @@ -1,122 +0,0 @@ -schema: 2 -name: dsv4-gb200-vllm-agentic-mtp-agg-tp8 - -model: - path: deepseek-v4-pro - container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - precision: fp4 - -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - frameworks: - dynamo: "1.3.1" - -dynamo: - install: true - - source: - pypi: "1.3.1" -setup_script: vllm-container-deps.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: gb200 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: kv - router-reset-states: true - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 256 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: /hf_hub_cache - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - args: - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 128 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - gpu-memory-utilization: 0.90 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - kv-cache-metrics: true - tokenizer-mode: deepseek_v4 - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml deleted file mode 100644 index 6e1cfc45eb..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml +++ /dev/null @@ -1,221 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-c128-mtp2-agentic" - -# GB200 AgentX MTP3 topology: one DEP8 prefill worker feeds one -# DEP8 decode worker at concurrency 128. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - install: true - - source: - wheel: "1.3.0.dev20260720" -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "140GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c128-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c128-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 128 - max-num-batched-tokens: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 512 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml deleted file mode 100644 index 4bd6b4ea1d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml +++ /dev/null @@ -1,221 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-c256-mtp2-agentic" - -# GB200 AgentX MTP3 topology: one DEP8 prefill worker feeds one -# DEP8 decode worker at concurrency 256. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - install: true - - source: - wheel: "1.3.0.dev20260720" -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "140GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 128 - max-num-batched-tokens: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 512 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml deleted file mode 100644 index 97eb57e1a0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml +++ /dev/null @@ -1,133 +0,0 @@ -schema: 2 -name: dsv4-gb200-vllm-agentic-mtp-disagg-1p1d-dep8-dep8 - -model: - path: deepseek-v4-pro - container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - precision: fp4 - -identity: - model: {repo: deepseek-ai/DeepSeek-V4-Pro} - container: {image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106} - frameworks: {dynamo: "1.3.1"} - -dynamo: - install: true - - source: - pypi: "1.3.1" -setup_script: vllm-container-deps.sh -environment: {ETCD_LEASE_TTL: "7200"} -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: gb200 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - # Long AgentX prefills can exceed Dynamo's request-plane default while - # the healthy DEP8 worker is still computing the first response. - DYN_TCP_REQUEST_TIMEOUT: "60" - args: - router-mode: kv - router-reset-states: true - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 256 - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: &worker_environment - HF_HUB_CACHE: /hf_hub_cache - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - args: &dep8_config - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - enforce-eager: true - block-size: 256 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - kv-cache-metrics: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: *worker_environment - args: - <<: *dep8_config - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' - enforce-eager: false - gpu-memory-utilization: 0.90 - max-num-seqs: 1024 - max-num-batched-tokens: 1024 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 - stream-interval: 10 - -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml deleted file mode 100644 index 57ab2522ce..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml +++ /dev/null @@ -1,223 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb200-2p1d-dep8-dep12-c576-mtp2-agentic" - -# GB200 AgentX MTP3 topology: two DEP8 prefill workers feed one -# DEP12 decode worker at concurrency 576. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - install: true - - source: - wheel: "1.3.0.dev20260720" -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "140GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 4 - workers: 2 - gpus: 8 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-prefill-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 3 - workers: 1 - gpus: 12 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-dep12-decode-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 12 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml deleted file mode 100644 index 908e5ad7ef..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml +++ /dev/null @@ -1,223 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb200-2p1d-dep8-dep16-c512-mtp2-agentic" - -# GB200 AgentX MTP3 topology: two DEP8 prefill workers feed one -# DEP16 decode worker at concurrency 512. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - install: true - - source: - wheel: "1.3.0.dev20260720" -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "140GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 4 - workers: 2 - gpus: 8 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-prefill-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-dep16-decode-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml deleted file mode 100644 index 63e789d9b9..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml +++ /dev/null @@ -1,130 +0,0 @@ -schema: 2 -name: dsv4-gb200-vllm-agentic-mtp-disagg-2p1d-dep8-dep8 - -model: - path: deepseek-v4-pro - container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - precision: fp4 - -identity: - model: {repo: deepseek-ai/DeepSeek-V4-Pro} - container: {image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106} - frameworks: {dynamo: "1.3.1"} - -dynamo: {install: true, source: {pypi: "1.3.1"}} -setup_script: vllm-container-deps.sh -environment: {ETCD_LEASE_TTL: "7200"} -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: gb200 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - # Long AgentX prefills can exceed Dynamo's request-plane default while - # the healthy DEP8 workers are still computing their first responses. - DYN_TCP_REQUEST_TIMEOUT: "60" - args: - router-mode: kv - router-reset-states: true - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 256 - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 4 - workers: 2 - gpus: 8 - env: &worker_environment - HF_HUB_CACHE: /hf_hub_cache - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - args: &dep8_config - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - enforce-eager: true - block-size: 256 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - kv-cache-metrics: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: *worker_environment - args: - <<: *dep8_config - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' - enforce-eager: false - gpu-memory-utilization: 0.90 - max-num-seqs: 1024 - max-num-batched-tokens: 1024 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 - stream-interval: 10 - -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml new file mode 100644 index 0000000000..57a3d6a417 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml @@ -0,0 +1,175 @@ +# AgentX dsv4 vllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml:override_. + +schema: 2 + +base: + model: + path: deepseek-v4-pro + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + precision: fp4 + identity: + model: {repo: deepseek-ai/DeepSeek-V4-Pro} + container: {image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106} + frameworks: {dynamo: "1.3.1"} + dynamo: + install: true + source: + pypi: "1.3.1" + setup_script: vllm-container-deps.sh + environment: {ETCD_LEASE_TTL: "7200"} + slurm: {time_limit: "8:00:00"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: gb200 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_TCP_REQUEST_TIMEOUT: "60" + args: + router-mode: kv + router-reset-states: true + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 256 + engine: + type: vllm + connector: + dp_launch_mode: per_node + roles: + prefill: + gpus: 8 + env: + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + enforce-eager: true + block-size: 256 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + kv-cache-metrics: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' + enforce-eager: false + gpu-memory-utilization: 0.90 + max-num-seqs: 1024 + max-num-batched-tokens: 1024 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + stream-interval: 10 + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + max-model-len: 1048576 + trust-remote-code: true + block-size: 256 + no-disable-hybrid-kv-cache-manager: true + kv-cache-metrics: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + sbatch_directives: {cpus-per-task: "144", mem: "0"} + srun_options: {container-remap-root: ""} + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + +# Long AgentX prefills can exceed Dynamo's request-plane default while +# the healthy DEP8 worker is still computing the first response. +override_1p1d: + name: dsv4-gb200-vllm-agentic-mtp-disagg-1p1d-dep8-dep8 + roles: + prefill: + nodes: 2 + workers: 1 + +# Long AgentX prefills can exceed Dynamo's request-plane default while +# the healthy DEP8 workers are still computing their first responses. +override_2p1d: + name: dsv4-gb200-vllm-agentic-mtp-disagg-2p1d-dep8-dep8 + roles: + prefill: + nodes: 4 + workers: 2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml new file mode 100644 index 0000000000..bfe51e70f6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml @@ -0,0 +1,303 @@ +# AgentX dsv4 vllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + dynamo: + install: true + source: + wheel: "1.3.0.dev20260720" + setup_script: vllm-container-deps.sh + slurm: + time_limit: "8:00:00" + health_check: + max_attempts: 2160 + interval_seconds: 10 + resources: + gpu_type: "gb200" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "140GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + engine: + type: vllm + connector: + dp_launch_mode: per_node + roles: + prefill: + gpus: 8 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + workers: 1 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + sbatch_directives: + cpus-per-task: "72" + srun_options: + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +# GB200 AgentX MTP3 topology: one DEP8 prefill worker feeds one +# DEP8 decode worker at concurrency 128. +override_1p1d_c128: + name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-c128-mtp2-agentic" + roles: + prefill: + nodes: 2 + workers: 1 + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c128-{job_id}" + args: + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.85 + decode: + nodes: 2 + gpus: 8 + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c128-{job_id}" + args: + data-parallel-size: 8 + max-num-seqs: 128 + max-num-batched-tokens: 1024 + max-cudagraph-capture-size: 512 + gpu-memory-utilization: 0.92 + +# GB200 AgentX MTP3 topology: one DEP8 prefill worker feeds one +# DEP8 decode worker at concurrency 256. +override_1p1d_c256: + name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-c256-mtp2-agentic" + roles: + prefill: + nodes: 2 + workers: 1 + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c256-{job_id}" + args: + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.85 + decode: + nodes: 2 + gpus: 8 + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c256-{job_id}" + args: + data-parallel-size: 8 + max-num-seqs: 128 + max-num-batched-tokens: 1024 + max-cudagraph-capture-size: 512 + gpu-memory-utilization: 0.92 + +# GB200 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP16 decode worker at concurrency 512. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. +override_2p1d_dep16_c512: + name: "svf-vllm-disagg-gb200-2p1d-dep8-dep16-c512-mtp2-agentic" + roles: + prefill: + nodes: 4 + workers: 2 + env: + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-prefill-{job_id}" + args: + gpu-memory-utilization: 0.90 + decode: + nodes: 4 + gpus: 16 + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-dep16-decode-{job_id}" + args: + data-parallel-size: 16 + max-num-seqs: 64 + max-num-batched-tokens: 256 + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.90 + sbatch_directives: + mem: "0" + +# GB200 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP12 decode worker at concurrency 576. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. +override_2p1d_dep12_c576: + name: "svf-vllm-disagg-gb200-2p1d-dep8-dep12-c576-mtp2-agentic" + roles: + prefill: + nodes: 4 + workers: 2 + env: + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-prefill-{job_id}" + args: + gpu-memory-utilization: 0.90 + decode: + nodes: 3 + gpus: 12 + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-dep12-decode-{job_id}" + args: + data-parallel-size: 12 + max-num-seqs: 64 + max-num-batched-tokens: 256 + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.90 + sbatch_directives: + mem: "0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml deleted file mode 100644 index 329626fbcc..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml +++ /dev/null @@ -1,159 +0,0 @@ -schema: 2 -name: "svf-vllm-agg-gb300-tp4-mtp-agentic" - -# GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node -# and serves both prefill and decode at concurrency 8. Size max-num-seqs at -# 4x concurrency and expand the MTP CUDA-graph envelope to match. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - install: true - - source: - wheel: "1.2.1" -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -engine: - type: vllm - connector: -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-mtp-{job_id}" - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml deleted file mode 100644 index e99921d95c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml +++ /dev/null @@ -1,160 +0,0 @@ -schema: 2 -name: "svf-vllm-agg-gb300-tp8-mtp-agentic" - -# GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU -# nodes and serves both prefill and decode at concurrency 4. Keep at least -# 16 sequence slots and otherwise size the scheduler at 4x concurrency. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - install: true - - source: - wheel: "1.2.1" -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - # The frontend shares Grace CPU capacity with the long TP8 cold start. - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -engine: - type: vllm - connector: -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-mtp-{job_id}" - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 16 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep aggregate workers in the multinode result schema so ingestion uses - # the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..69ec5c40ff --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml @@ -0,0 +1,182 @@ +# AgentX dsv4 vllm gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.2.1" + dynamo: + install: true + source: + wheel: "1.2.1" + environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + # The frontend shares Grace CPU capacity with the long TP8 cold start. + ETCD_LEASE_TTL: "7200" + setup_script: vllm-container-deps.sh + slurm: + time_limit: "8:00:00" + health_check: + max_attempts: 2160 + interval_seconds: 10 + resources: + gpu_type: "gb300" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + engine: + type: vllm + connector: + roles: + agg: + workers: 1 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + gpu-memory-utilization: 0.94 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + sbatch_directives: + mem: "0" + srun_options: + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep aggregate workers in the multinode result schema so ingestion uses + # the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +# GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node +# and serves both prefill and decode at concurrency 8. Size max-num-seqs at +# 4x concurrency and expand the MTP CUDA-graph envelope to match. +override_tp4: + name: "svf-vllm-agg-gb300-tp4-mtp-agentic" + roles: + agg: + nodes: 1 + gpus: 4 + env: + VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-mtp-{job_id}" + args: + tensor-parallel-size: 4 + max-num-seqs: 32 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' + sbatch_directives: + cpus-per-task: "72" + benchmark: + env: + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + +# GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU +# nodes and serves both prefill and decode at concurrency 4. Keep at least +# 16 sequence slots and otherwise size the scheduler at 4x concurrency. +override_tp8: + name: "svf-vllm-agg-gb300-tp8-mtp-agentic" + roles: + agg: + nodes: 2 + gpus: 8 + env: + VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-mtp-{job_id}" + args: + tensor-parallel-size: 8 + max-num-seqs: 16 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' + sbatch_directives: + cpus-per-task: "144" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml deleted file mode 100644 index 96d135c0f7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml +++ /dev/null @@ -1,228 +0,0 @@ -# Derived from PR #2302's DEP8/DEP32 c256 recipe. -# Halve the prefill/decode topology while preserving its conservative -# 4/16/16 decode limits and isolated JIT cache paths. -schema: 2 -name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c128-mtp-agentic" - -# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one -# DEP16 decode worker at concurrency 128. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.4.0" - -dynamo: - install: true - - source: - wheel: "1.4.0" -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c128-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c128-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 4 - max-num-batched-tokens: 16 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 16 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml deleted file mode 100644 index c91da9bb8e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml +++ /dev/null @@ -1,228 +0,0 @@ -# Derived from PR #2302's DEP8/DEP32 c256 recipe. -# Halve the prefill/decode topology while preserving its conservative -# 8/32/32 decode limits and isolated JIT cache paths. -schema: 2 -name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic" - -# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one -# DEP16 decode worker at concurrency 256. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.4.0" - -dynamo: - install: true - - source: - wheel: "1.4.0" -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 8 - max-num-batched-tokens: 32 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 32 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml deleted file mode 100644 index 130997107c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml +++ /dev/null @@ -1,221 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic" - -# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one -# DEP8 decode worker at concurrency 256. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - install: true - - source: - wheel: "1.3.0.dev20260720" -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml deleted file mode 100644 index 3f91ed3e2d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml +++ /dev/null @@ -1,228 +0,0 @@ -# Source: https://github.com/SemiAnalysisAI/InferenceX/blob/0c33d4615792705ed12bfc204e3a54cfa436cf02/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml -# Runtime and performance arguments follow the current DEP8/DEP16 GB300 P/D -# recipes; throughput-only synthetic MTP acceptance is injected at launch. -schema: 2 -name: "svf-vllm-disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic" - -# GB300 AgentX MTP3 topology: one DEP8 prefill worker feeds one -# DEP16 decode worker at concurrency 512. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.4.0" - -dynamo: - install: true - - source: - wheel: "1.4.0" -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c512-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 16384 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep16-decode-c512-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml deleted file mode 100644 index 82256d088c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb300-1p6d-dep4-tp4-agentic" - -# Agentic-coding variant of dsv4/vllm/gb300-fp4/8k1k/disagg-1p6d-dep4-tp4-stp.yaml. -# Topology is identical (1 prefill DEP=4 + 6 decode TP=4, 28 GPUs across 7 -# GB300 nodes + 1 dedicated NATS/etcd infra node) so we can compare against -# the fixed-seq-len 1p6d baseline at the same concurrency point (192). -# -# Divergence vs the 8k1k sibling: -# - benchmark.type: sa-bench -> custom (hands off to srt_agentic.sh) -# - max-model-len: removed (let vLLM derive from model config; agentic -# trajectories blow past any small explicit cap) -# - no-enable-prefix-caching: dropped (prefix caching MUST be on for -# trajectory reuse — entire point of agentic) -# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser -# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't -# accept them (different arg parser than `vllm serve`). In disagg, chat -# parsing happens at the dynamo frontend, not at the worker. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.21.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - source: - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - attention-config: '{"use_fp4_indexer_cache": true}' - moe-backend: "deep_gemm_mega_moe" - # enforce-eager: true - # max-num-seqs: 256 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.9 - enable-ep-weight-filter: true - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - decode: - nodes: 6 - workers: 6 - gpus: 4 - - env: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - # max-num-seqs: 512 - trust-remote-code: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - enable-ep-weight-filter: true - all2all-backend: "flashinfer_nvlink_one_sided" - no-enable-flashinfer-autotune: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - -# sbatch + srun resource grants for clusters without per-GPU defaults. -# -# mem=0: allocate all available node memory (~868 GB on CW gb300). Without -# this, sbatch only requests ntasks × DefMemPerCPU = 8 × 4 GB = 32 GB for -# the whole job and worker cgroups OOM-kill mid model load (R7-R11 hit -# this; sacct showed AllocTRES mem=4G per step). -# -# cpus-per-task=72: give each task one CW gb300 NUMA socket (144 cores -# split 2 × 72). Critical for the *infra step* (etcd + nats) which -# srtctl spawns without --gres=gpu — on CW that means DefMemPerCPU -# applies and the step gets 1 CPU by default. With 24 dynamo DP ranks -# all hammering etcd for lease keep-alives, single-CPU etcd can't keep -# up and dies (R12 hit this; etcd reported max-cpu-set=1, leases -# deadline-exceeded, infra SIGKILL'd at 16:35:49). 72 CPUs is plenty -# for both etcd + nats AND for vLLM worker auxiliary threads. -# -# nv gb300 doesn't need this because cluster default DefCpuPerGPU=35 -# auto-allocates 4*35=140 CPUs per GPU-bearing task; cw has no per-GPU -# default. Setting it here is safe on both because the value is ≤ node -# CPU count. -# -# srun_options.mem=0 forces each srun step to use the full node memory -# (without it, srun steps default back to cpus_per_task × DefMemPerCPU). -# Docs: docs/config-reference.md#sbatch_directives + #srun_options. -sbatch_directives: - mem: "0" - cpus-per-task: "72" -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user (sa-shared) into the container as - # uid 345200007. dpkg refuses to run without EUID 0 even though - # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the srt_agentic - # apt-get install git step fails. --container-remap-root asks pyxis to - # remap us to uid 0 inside the container. srt-slurm renders empty-string - # values as flag-only srun args (see core/slurm.py:250). - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - # Container-side path of the aiperf mmap dataset cache; the host-side - # mount is wired via launch_gb300-*.sh's srtslurm.yaml default_mounts. - # Without this, aiperf re-tokenizes + re-writes ~65 GB of mmap files - # per dataset on every run. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also wired via default_mounts) so the trace - # dataset isn't re-downloaded on every run. Overrides the workflow-level - # HF_HUB_CACHE=/mnt/hf_hub_cache, which doesn't exist on these nodes. - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml deleted file mode 100644 index 9e35de536e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml +++ /dev/null @@ -1,223 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic" - -# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one -# DEP12 decode worker at concurrency 1152. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - install: true - - source: - wheel: "1.3.0.dev20260720" -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 4 - workers: 2 - gpus: 8 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 3 - workers: 1 - gpus: 12 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep12-decode-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 12 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml deleted file mode 100644 index 2e6ab175fc..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml +++ /dev/null @@ -1,223 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic" - -# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one -# DEP16 decode worker at concurrency 1024. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - install: true - - source: - wheel: "1.3.0.dev20260720" -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 4 - workers: 2 - gpus: 8 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep16-decode-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml deleted file mode 100644 index 08766587b8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml +++ /dev/null @@ -1,188 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic" - -# Agentic-coding variant of dsv4/vllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep8-24-c4096-stp.yaml. -# Max-throughput shape: 4 prefill (DEP=4 each) + 1 decode (DEP=8). 6 GB300 -# nodes (4P + 2D = 24 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra -# node. Sized for concurrency 4096 with deep_gemm_mega_moe on both workers. -# -# Divergence vs the 8k1k sibling: -# - benchmark.type: sa-bench -> custom (hands off to srt_agentic.sh) -# - max-model-len: removed (let vLLM derive from model config; agentic -# trajectories blow past any small explicit cap) -# - no-enable-prefix-caching: dropped (prefix caching MUST be on for -# trajectory reuse — entire point of agentic) -# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser -# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't -# accept them (different arg parser than `vllm serve`). In disagg, chat -# parsing happens at the dynamo frontend, not at the worker. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.21.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - source: - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 4 - workers: 4 - gpus: 4 - env: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - # enforce-eager: true - # Inherited from fixed-sequence recipes; let vLLM select the scheduler - # sequence limit until this is tuned explicitly for the agentic trace. - # max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-flashinfer-autotune: true - safetensors-load-strategy: "prefetch" - block-size: 256 - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_mega_moe" - - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - # max-num-seqs: 512 - trust-remote-code: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_mega_moe" - -# sbatch + srun resource grants for clusters without per-GPU defaults. -# -# mem=0: allocate all available node memory (~868 GB on CW gb300). Without -# this, sbatch only requests ntasks × DefMemPerCPU = 8 × 4 GB = 32 GB for -# the whole job and worker cgroups OOM-kill mid model load (R7-R11 hit -# this; sacct showed AllocTRES mem=4G per step). -# -# cpus-per-task=72: give each task one CW gb300 NUMA socket (144 cores -# split 2 × 72). Critical for the *infra step* (etcd + nats) which -# srtctl spawns without --gres=gpu — on CW that means DefMemPerCPU -# applies and the step gets 1 CPU by default. With 24 dynamo DP ranks -# all hammering etcd for lease keep-alives, single-CPU etcd can't keep -# up and dies (R12 hit this; etcd reported max-cpu-set=1, leases -# deadline-exceeded, infra SIGKILL'd at 16:35:49). 72 CPUs is plenty -# for both etcd + nats AND for vLLM worker auxiliary threads. -# -# nv gb300 doesn't need this because cluster default DefCpuPerGPU=35 -# auto-allocates 4*35=140 CPUs per GPU-bearing task; cw has no per-GPU -# default. Setting it here is safe on both because the value is ≤ node -# CPU count. -# -# srun_options.mem=0 forces each srun step to use the full node memory -# (without it, srun steps default back to cpus_per_task × DefMemPerCPU). -# Docs: docs/config-reference.md#sbatch_directives + #srun_options. -sbatch_directives: - mem: "0" - cpus-per-task: "72" -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user (sa-shared) into the container as - # uid 345200007. dpkg refuses to run without EUID 0 even though - # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the srt_agentic - # apt-get install git step fails. --container-remap-root asks pyxis to - # remap us to uid 0 inside the container. srt-slurm renders empty-string - # values as flag-only srun args (see core/slurm.py:250). - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - # Container-side path of the aiperf mmap dataset cache; the host-side - # mount is wired via launch_gb300-*.sh's srtslurm.yaml default_mounts. - # Without this, aiperf re-tokenizes + re-writes ~65 GB of mmap files - # per dataset on every run. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also wired via default_mounts) so the trace - # dataset isn't re-downloaded on every run. Overrides the workflow-level - # HF_HUB_CACHE=/mnt/hf_hub_cache, which doesn't exist on these nodes. - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml new file mode 100644 index 0000000000..c91f01d08a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml @@ -0,0 +1,414 @@ +# AgentX dsv4 vllm gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + dynamo: + install: true + environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + setup_script: vllm-container-deps.sh + slurm: + time_limit: "8:00:00" + health_check: + max_attempts: 2160 + interval_seconds: 10 + resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + engine: + type: vllm + connector: + dp_launch_mode: per_node + roles: + prefill: + workers: 1 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + sbatch_directives: + cpus-per-task: "72" + srun_options: + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +# Derived from PR #2302's DEP8/DEP32 c256 recipe. +# Halve the prefill/decode topology while preserving its conservative +# 4/16/16 decode limits and isolated JIT cache paths. +# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one +# DEP16 decode worker at concurrency 128. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. +override_1p1d_dep4_dep16_c128: + name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c128-mtp-agentic" + identity: + frameworks: + dynamo: "1.4.0" + dynamo: + source: + wheel: "1.4.0" + roles: + prefill: + nodes: 1 + gpus: 4 + env: + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c128-{job_id}" + args: + safetensors-load-strategy: "prefetch" + data-parallel-size: 4 + max-num-seqs: 256 + decode: + env: + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c128-{job_id}" + args: + safetensors-load-strategy: "prefetch" + max-num-seqs: 4 + max-num-batched-tokens: 16 + max-cudagraph-capture-size: 16 + sbatch_directives: + mem: "0" + +# Derived from PR #2302's DEP8/DEP32 c256 recipe. +# Halve the prefill/decode topology while preserving its conservative +# 8/32/32 decode limits and isolated JIT cache paths. +# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one +# DEP16 decode worker at concurrency 256. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. +override_1p1d_dep4_dep16_c256: + name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic" + identity: + frameworks: + dynamo: "1.4.0" + dynamo: + source: + wheel: "1.4.0" + roles: + prefill: + nodes: 1 + gpus: 4 + env: + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c256-{job_id}" + args: + safetensors-load-strategy: "prefetch" + data-parallel-size: 4 + max-num-seqs: 256 + decode: + env: + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c256-{job_id}" + args: + safetensors-load-strategy: "prefetch" + max-num-seqs: 8 + max-num-batched-tokens: 32 + max-cudagraph-capture-size: 32 + sbatch_directives: + mem: "0" + +# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one +# DEP8 decode worker at concurrency 256. +override_1p1d_dep4_dep8_c256: + name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic" + identity: + frameworks: + dynamo: "1.3.0.dev20260720" + dynamo: + source: + wheel: "1.3.0.dev20260720" + roles: + prefill: + nodes: 1 + gpus: 4 + env: + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" + args: + data-parallel-size: 4 + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.92 + decode: + nodes: 2 + gpus: 8 + env: + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" + args: + data-parallel-size: 8 + max-num-seqs: 256 + max-num-batched-tokens: 1024 + max-cudagraph-capture-size: 1024 + gpu-memory-utilization: 0.92 + +# Source: https://github.com/SemiAnalysisAI/InferenceX/blob/0c33d4615792705ed12bfc204e3a54cfa436cf02/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml +# Runtime and performance arguments follow the current DEP8/DEP16 GB300 P/D +# recipes; throughput-only synthetic MTP acceptance is injected at launch. +# GB300 AgentX MTP3 topology: one DEP8 prefill worker feeds one +# DEP16 decode worker at concurrency 512. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. +override_1p1d_dep8_dep16_c512: + name: "svf-vllm-disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic" + identity: + frameworks: + dynamo: "1.4.0" + dynamo: + source: + wheel: "1.4.0" + roles: + prefill: + nodes: 2 + gpus: 8 + env: + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c512-{job_id}" + args: + safetensors-load-strategy: "prefetch" + data-parallel-size: 8 + max-num-batched-tokens: 16384 + decode: + env: + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep16-decode-c512-{job_id}" + args: + safetensors-load-strategy: "prefetch" + max-num-seqs: 64 + max-num-batched-tokens: 256 + max-cudagraph-capture-size: 256 + sbatch_directives: + mem: "0" + +# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP16 decode worker at concurrency 1024. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. +override_2p1d_dep8_dep16_c1024: + name: "svf-vllm-disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic" + identity: + frameworks: + dynamo: "1.3.0.dev20260720" + dynamo: + source: + wheel: "1.3.0.dev20260720" + roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + env: + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" + args: + data-parallel-size: 8 + decode: + env: + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep16-decode-{job_id}" + args: + max-num-seqs: 64 + max-num-batched-tokens: 256 + max-cudagraph-capture-size: 256 + sbatch_directives: + mem: "0" + +# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP12 decode worker at concurrency 1152. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. +override_2p1d_dep8_dep12_c1152: + name: "svf-vllm-disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic" + identity: + frameworks: + dynamo: "1.3.0.dev20260720" + dynamo: + source: + wheel: "1.3.0.dev20260720" + roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + env: + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" + args: + data-parallel-size: 8 + decode: + nodes: 3 + gpus: 12 + env: + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep12-decode-{job_id}" + args: + data-parallel-size: 12 + max-num-seqs: 64 + max-num-batched-tokens: 256 + max-cudagraph-capture-size: 256 + sbatch_directives: + mem: "0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..fc0aae1a06 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,222 @@ +# AgentX dsv4 vllm gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0-ubuntu2404" + precision: "fp4" + dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + setup_script: vllm-container-deps.sh + slurm: + time_limit: "8:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: "gb300" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 + frontend: + type: dynamo + enable_multiple_frontends: false + engine: + type: vllm + connector: + roles: + prefill: + gpus: 4 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + moe-backend: "deep_gemm_mega_moe" + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + enable-ep-weight-filter: true + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + decode: + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + pipeline-parallel-size: 1 + # max-num-seqs: 512 + trust-remote-code: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + enable-ep-weight-filter: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + # sbatch + srun resource grants for clusters without per-GPU defaults. + # + # mem=0: allocate all available node memory (~868 GB on CW gb300). Without + # this, sbatch only requests ntasks × DefMemPerCPU = 8 × 4 GB = 32 GB for + # the whole job and worker cgroups OOM-kill mid model load (R7-R11 hit + # this; sacct showed AllocTRES mem=4G per step). + # + # cpus-per-task=72: give each task one CW gb300 NUMA socket (144 cores + # split 2 × 72). Critical for the *infra step* (etcd + nats) which + # srtctl spawns without --gres=gpu — on CW that means DefMemPerCPU + # applies and the step gets 1 CPU by default. With 24 dynamo DP ranks + # all hammering etcd for lease keep-alives, single-CPU etcd can't keep + # up and dies (R12 hit this; etcd reported max-cpu-set=1, leases + # deadline-exceeded, infra SIGKILL'd at 16:35:49). 72 CPUs is plenty + # for both etcd + nats AND for vLLM worker auxiliary threads. + # + # nv gb300 doesn't need this because cluster default DefCpuPerGPU=35 + # auto-allocates 4*35=140 CPUs per GPU-bearing task; cw has no per-GPU + # default. Setting it here is safe on both because the value is ≤ node + # CPU count. + # + # srun_options.mem=0 forces each srun step to use the full node memory + # (without it, srun steps default back to cpus_per_task × DefMemPerCPU). + # Docs: docs/config-reference.md#sbatch_directives + #srun_options. + sbatch_directives: + mem: "0" + cpus-per-task: "72" + srun_options: + mem: "0" + # gb300-nv: pyxis maps the calling user (sa-shared) into the container as + # uid 345200007. dpkg refuses to run without EUID 0 even though + # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the srt_agentic + # apt-get install git step fails. --container-remap-root asks pyxis to + # remap us to uid 0 inside the container. srt-slurm renders empty-string + # values as flag-only srun args (see core/slurm.py:250). + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + # Container-side path of the aiperf mmap dataset cache; the host-side + # mount is wired via launch_gb300-*.sh's srtslurm.yaml default_mounts. + # Without this, aiperf re-tokenizes + re-writes ~65 GB of mmap files + # per dataset on every run. + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + # Persistent HF hub cache (also wired via default_mounts) so the trace + # dataset isn't re-downloaded on every run. Overrides the workflow-level + # HF_HUB_CACHE=/mnt/hf_hub_cache, which doesn't exist on these nodes. + HF_HUB_CACHE: "/hf_hub_cache" + +# Agentic-coding variant of dsv4/vllm/gb300-fp4/8k1k/disagg-1p6d-dep4-tp4-stp.yaml. +# Topology is identical (1 prefill DEP=4 + 6 decode TP=4, 28 GPUs across 7 +# GB300 nodes + 1 dedicated NATS/etcd infra node) so we can compare against +# the fixed-seq-len 1p6d baseline at the same concurrency point (192). +# +# Divergence vs the 8k1k sibling: +# - benchmark.type: sa-bench -> custom (hands off to srt_agentic.sh) +# - max-model-len: removed (let vLLM derive from model config; agentic +# trajectories blow past any small explicit cap) +# - no-enable-prefix-caching: dropped (prefix caching MUST be on for +# trajectory reuse — entire point of agentic) +# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser +# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't +# accept them (different arg parser than `vllm serve`). In disagg, chat +# parsing happens at the dynamo frontend, not at the worker. +override_1p6d_dep4_tp4: + name: "svf-vllm-disagg-gb300-1p6d-dep4-tp4-agentic" + roles: + prefill: + nodes: 1 + workers: 1 + args: + # enforce-eager: true + # max-num-seqs: 256 + attention-config: '{"use_fp4_indexer_cache": true}' + decode: + nodes: 6 + workers: 6 + gpus: 4 + args: + tensor-parallel-size: 4 + all2all-backend: "flashinfer_nvlink_one_sided" + no-enable-flashinfer-autotune: true + +# Agentic-coding variant of dsv4/vllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep8-24-c4096-stp.yaml. +# Max-throughput shape: 4 prefill (DEP=4 each) + 1 decode (DEP=8). 6 GB300 +# nodes (4P + 2D = 24 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra +# node. Sized for concurrency 4096 with deep_gemm_mega_moe on both workers. +# +# Divergence vs the 8k1k sibling: +# - benchmark.type: sa-bench -> custom (hands off to srt_agentic.sh) +# - max-model-len: removed (let vLLM derive from model config; agentic +# trajectories blow past any small explicit cap) +# - no-enable-prefix-caching: dropped (prefix caching MUST be on for +# trajectory reuse — entire point of agentic) +# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser +# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't +# accept them (different arg parser than `vllm serve`). In disagg, chat +# parsing happens at the dynamo frontend, not at the worker. +override_4p1d_dep4_dep8_24_c4096: + name: "svf-vllm-disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic" + roles: + prefill: + nodes: 4 + workers: 4 + env: + VLLM_LOG_STATS_INTERVAL: "1" + args: + # enforce-eager: true + # Inherited from fixed-sequence recipes; let vLLM select the scheduler + # sequence limit until this is tuned explicitly for the agentic trace. + # max-num-seqs: 16 + safetensors-load-strategy: "prefetch" + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + VLLM_LOG_STATS_INTERVAL: "1" + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + moe-backend: "deep_gemm_mega_moe" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 99ee1b3432..847d449d74 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5153,7 +5153,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p6d_dep4_tp4" decode: num-worker: 6 tp: 4 @@ -5169,7 +5169,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p6d_dep4_tp4" decode: num-worker: 6 tp: 4 @@ -5186,7 +5186,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_4p1d_dep4_dep8_24_c4096" decode: num-worker: 1 tp: 8 @@ -5978,7 +5978,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml:override_tp8" - spec-decoding: mtp conc-list: [8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80] num-nodes: 2 @@ -5988,7 +5988,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml:override_dep8" dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 model: deepseek-ai/DeepSeek-V4-Pro @@ -6012,7 +6012,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml:override_1p1d" decode: num-worker: 1 tp: 8 @@ -6027,7 +6027,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml:override_2p1d" decode: num-worker: 1 tp: 8 @@ -6058,7 +6058,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml:override_tp8" - spec-decoding: mtp conc-list: [4] num-nodes: 2 @@ -6068,7 +6068,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml:override_tp8" - spec-decoding: mtp conc-list: [1, 2, 4, 6, 8] num-nodes: 1 @@ -6078,7 +6078,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml:override_tp4" dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro @@ -6107,7 +6107,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_2p1d_dep8_dep12_c1152" decode: num-worker: 1 tp: 12 @@ -6124,7 +6124,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_2p1d_dep8_dep16_c1024" decode: num-worker: 1 tp: 16 @@ -6141,7 +6141,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_1p1d_dep4_dep8_c256" decode: num-worker: 1 tp: 8 @@ -6158,7 +6158,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_1p1d_dep8_dep16_c512" decode: num-worker: 1 tp: 16 @@ -6190,7 +6190,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_1p1d_dep4_dep16_c128" decode: num-worker: 1 tp: 16 @@ -6207,7 +6207,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_1p1d_dep4_dep16_c256" decode: num-worker: 1 tp: 16 @@ -6243,7 +6243,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml:override_c4" - spec-decoding: mtp conc-list: [4] num-nodes: 2 @@ -6253,7 +6253,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml:override_c4" - spec-decoding: mtp conc-list: [8] num-nodes: 2 @@ -6263,7 +6263,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml:override_c8" dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro @@ -6292,7 +6292,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml:override_2p1d_dep12_c576" decode: num-worker: 1 tp: 12 @@ -6309,7 +6309,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml:override_2p1d_dep16_c512" decode: num-worker: 1 tp: 16 @@ -6326,7 +6326,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml:override_1p1d_c256" decode: num-worker: 1 tp: 8 @@ -6343,7 +6343,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml:override_1p1d_c128" decode: num-worker: 1 tp: 8 @@ -6506,7 +6506,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p4d_dep4_tep8_c4_b1" - "SLURM_PARTITION=batch_1" decode: num-worker: 4 @@ -6523,7 +6523,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p6d_dep4_tep4_c24_b4" - "SLURM_PARTITION=batch_1" decode: num-worker: 6 @@ -6540,7 +6540,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_dep8_dep32_c388_b4" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -6557,7 +6557,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p1d_dep8_dep32_c736_b8" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -6574,7 +6574,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_3p1d_dep8_dep16_c1152_b32" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -6591,7 +6591,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_5p1d_dep8_dep16_c2626_b96" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -6702,7 +6702,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml:override_tp8" - search-space: - spec-decoding: draft_model conc-list: [8] @@ -6713,7 +6713,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml:override_tp4" dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21 model: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -6739,7 +6739,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_c480" decode: num-worker: 1 tp: 16 @@ -6755,7 +6755,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_2p1d_c960" decode: num-worker: 1 tp: 16 @@ -6771,7 +6771,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_3p1d_c1440" decode: num-worker: 1 tp: 16 @@ -6787,7 +6787,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_4p1d_c1920" decode: num-worker: 1 tp: 16 @@ -8574,7 +8574,7 @@ dsv4-fp4-b200-dynamo-sglang-agentic-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c1-mtp-hicache.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml:override_c1" - spec-decoding: mtp conc-list: [4] kv-offloading: dram @@ -8588,7 +8588,7 @@ dsv4-fp4-b200-dynamo-sglang-agentic-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c4-mtp-hicache.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml:override_c4" - spec-decoding: mtp conc-list: [8] kv-offloading: dram @@ -8602,7 +8602,7 @@ dsv4-fp4-b200-dynamo-sglang-agentic-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c8-mtp-hicache.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml:override_c8" dsv4-fp4-b200-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 @@ -8631,7 +8631,7 @@ dsv4-fp4-b200-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml:override_1p1d_c64" decode: num-worker: 1 tp: 8 @@ -8649,7 +8649,7 @@ dsv4-fp4-b200-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml:override_1p1d_c128" decode: num-worker: 1 tp: 8 @@ -8667,7 +8667,7 @@ dsv4-fp4-b200-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml:override_2p1d_c256" decode: num-worker: 1 tp: 8 From c08474c6de6885ca4564554eb2f9ba4132f60838 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:37:32 -0500 Subject: [PATCH 073/105] refactor(agentx): consolidate GLM-5.2 multi-node AgentX recipes into override variants Each per-configuration recipe becomes an `override_` block over a shared `base` in one `*-variants.yaml` per master-config entry, and the master entries select it with `CONFIG_FILE=...:override_`. Every selected variant resolves, through the pinned srtctl, to exactly the recipe it replaces, including its original `name`. Power recipes with top-level telemetry stay standalone because launchers detect them as text. --- .../b200-fp4/agentx/agg-tp8-c1-mtp.yaml | 124 ------ .../b200-fp4/agentx/agg-tp8-c4-mtp.yaml | 124 ------ .../b200-fp4/agentx/agg-tp8-c8-mtp.yaml | 124 ------ .../sglang/b200-fp4/agentx/agg-variants.yaml | 163 ++++++++ .../agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml | 198 --------- .../agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml | 190 --------- .../b200-fp4/agentx/disagg-variants.yaml | 238 +++++++++++ .../gb200-fp4/agentx/agg-mtp-variants.yaml | 142 +++++++ .../gb200-fp4/agentx/agg-tp8-c2-mtp.yaml | 128 ------ .../gb200-fp4/agentx/agg-tp8-c4-mtp.yaml | 128 ------ .../gb200-fp4/agentx/agg-tp8-c8-mtp.yaml | 128 ------ .../agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml | 199 --------- .../agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml | 199 --------- .../disagg-2p1d-dep8-dep16-c128-mtp.yaml | 207 ---------- .../agentx/disagg-dep8-mtp-variants.yaml | 247 ++++++++++++ .../agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml | 188 --------- .../agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml | 185 --------- .../agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml | 188 --------- .../agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml | 188 --------- .../disagg-4p1d-dep8-c227-b16-mtp3.yaml | 189 --------- .../disagg-5p1d-dep16-c260-b16-mtp3.yaml | 189 --------- .../gb300-fp4/agentx/disagg-variants.yaml | 380 ++++++++++++++++++ configs/nvidia-master.yaml | 34 +- 23 files changed, 1187 insertions(+), 2893 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml deleted file mode 100644 index fe123e03bd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml +++ /dev/null @@ -1,124 +0,0 @@ -# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 1. -# -# One TP8 worker serves prefill and decode on a single node. The sweep matrix -# supplies the concurrency list; srt_agentic.sh replays every point against -# this one server. Acceptance is pinned to the golden thinking-on AL for three -# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). -schema: 2 -name: agg-b200-tp8-c1-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 - frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 -resources: - gpu_type: b200 - gpus_per_node: 8 -dynamo: - install: false -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 2 - cuda-graph-max-bs-decode: 2 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 110 - hicache-io-backend: direct - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -health_check: - max_attempts: 1440 - interval_seconds: 10 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -frontend: - type: dynamo - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-session-affinity-ttl-secs: 3600 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml deleted file mode 100644 index 4fdb449e44..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml +++ /dev/null @@ -1,124 +0,0 @@ -# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 4. -# -# One TP8 worker serves prefill and decode on a single node. The sweep matrix -# supplies the concurrency list; srt_agentic.sh replays every point against -# this one server. Acceptance is pinned to the golden thinking-on AL for three -# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). -schema: 2 -name: agg-b200-tp8-c4-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 - frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 -resources: - gpu_type: b200 - gpus_per_node: 8 -dynamo: - install: false -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 8 - cuda-graph-max-bs-decode: 8 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 110 - hicache-io-backend: direct - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -health_check: - max_attempts: 1440 - interval_seconds: 10 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -frontend: - type: dynamo - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-session-affinity-ttl-secs: 3600 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml deleted file mode 100644 index 6b0cbfa53a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml +++ /dev/null @@ -1,124 +0,0 @@ -# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 8. -# -# One TP8 worker serves prefill and decode on a single node. The sweep matrix -# supplies the concurrency list; srt_agentic.sh replays every point against -# this one server. Acceptance is pinned to the golden thinking-on AL for three -# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). -schema: 2 -name: agg-b200-tp8-c8-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 - frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 -resources: - gpu_type: b200 - gpus_per_node: 8 -dynamo: - install: false -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 110 - hicache-io-backend: direct - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -health_check: - max_attempts: 1440 - interval_seconds: 10 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -frontend: - type: dynamo - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-session-affinity-ttl-secs: 3600 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..f7b88919e4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml @@ -0,0 +1,163 @@ +# AgentX glm5.2 sglang b200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 + identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + frameworks: + dynamo: 1.5.0.dev20260909 + sglang: 0.0.0.dev1+g008403017 + resources: + gpu_type: b200 + gpus_per_node: 8 + dynamo: + install: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 110 + hicache-io-backend: direct + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + health_check: + max_attempts: 1440 + interval_seconds: 10 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 + frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 + sbatch_directives: + mem: '0' + srun_options: + mem: '0' + container-remap-root: '' + +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 1. +# +# One TP8 worker serves prefill and decode on a single node. The sweep matrix +# supplies the concurrency list; srt_agentic.sh replays every point against +# this one server. Acceptance is pinned to the golden thinking-on AL for three +# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). +override_c1: + name: agg-b200-tp8-c1-mtp + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 4. +# +# One TP8 worker serves prefill and decode on a single node. The sweep matrix +# supplies the concurrency list; srt_agentic.sh replays every point against +# this one server. Acceptance is pinned to the golden thinking-on AL for three +# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). +override_c4: + name: agg-b200-tp8-c4-mtp + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 8. +# +# One TP8 worker serves prefill and decode on a single node. The sweep matrix +# supplies the concurrency list; srt_agentic.sh replays every point against +# this one server. Acceptance is pinned to the golden thinking-on AL for three +# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). +override_c8: + name: agg-b200-tp8-c8-mtp + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml deleted file mode 100644 index f9b90266de..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml +++ /dev/null @@ -1,198 +0,0 @@ -schema: 2 -name: disagg-b200-1p1d-dep8-dep8-c64-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 - frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 -resources: - gpu_type: b200 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: false -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 8192 - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 110 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true - decode: - nodes: 1 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 128 - cuda-graph-max-bs-decode: 128 - chunked-prefill-size: 64 - context-length: 1048576 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutedsl - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.85 - disaggregation-decode-extra-slots: 0 - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true - deepep-config: /configs/deepep_config.json - deepep-mode: low_latency - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - speculative-moe-a2a-backend: deepep - speculative-moe-runner-backend: deep_gemm -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml deleted file mode 100644 index 505c4935bf..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml +++ /dev/null @@ -1,190 +0,0 @@ -schema: 2 -name: disagg-b200-1p4d-dep8-tp4-c48-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 - frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 -resources: - gpu_type: b200 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: false -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '3600' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 110 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true - decode: - nodes: 2 - workers: 4 - gpus: 4 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '3600' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 - chunked-prefill-size: 64 - context-length: 1048576 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.9 - disaggregation-decode-extra-slots: 0 - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..02a84e30e1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,238 @@ +# AgentX glm5.2 sglang b200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 + identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + frameworks: + dynamo: 1.5.0.dev20260909 + sglang: 0.0.0.dev1+g008403017 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + dynamo: + install: false + engine: sglang + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 110 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + decode: + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + chunked-prefill-size: 64 + context-length: 1048576 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + disaggregation-decode-extra-slots: 0 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + health_check: + max_attempts: 1440 + interval_seconds: 10 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 + sbatch_directives: + mem: '0' + srun_options: + mem: '0' + container-remap-root: '' + +override_1p4d_tp4_c48: + name: disagg-b200-1p4d-dep8-tp4-c48-mtp + roles: + prefill: + env: + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '3600' + args: + max-prefill-tokens: 16384 + decode: + nodes: 2 + workers: 4 + gpus: 4 + env: + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '3600' + args: + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + moe-runner-backend: flashinfer_trtllm + mem-fraction-static: 0.9 + +override_1p1d_c64: + name: disagg-b200-1p1d-dep8-dep8-c64-mtp + roles: + prefill: + env: + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' + args: + max-prefill-tokens: 8192 + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' + args: + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 128 + cuda-graph-max-bs-decode: 128 + moe-runner-backend: flashinfer_cutedsl + mem-fraction-static: 0.85 + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + speculative-moe-a2a-backend: deepep + speculative-moe-runner-backend: deep_gemm diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml new file mode 100644 index 0000000000..a9c3b602b3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml @@ -0,0 +1,142 @@ +# AgentX glm5.2 sglang gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml:override_. + +schema: 2 + +base: + model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 + identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + frameworks: + dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + sglang: nightly-dev-cu13-20260805-211ee642 + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + dynamo: + install: true + source: + rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + engine: sglang + roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 10 + cuda-graph-max-bs: 10 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 4 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 5 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 135 + hicache-io-backend: direct + enable-metrics: true + health_check: + max_attempts: 1440 + interval_seconds: 10 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + +override_c2: + name: agg-gb200-tp8-c2-mtp + +override_c4: + name: agg-gb200-tp8-c4-mtp + +override_c8: + name: agg-gb200-tp8-c8-mtp diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml deleted file mode 100644 index e0a6fcaec6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml +++ /dev/null @@ -1,128 +0,0 @@ -schema: 2 -name: agg-gb200-tp8-c2-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: true - source: - rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd -engine: sglang -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 10 - cuda-graph-max-bs: 10 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - enable-metrics: true -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml deleted file mode 100644 index c192760a10..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml +++ /dev/null @@ -1,128 +0,0 @@ -schema: 2 -name: agg-gb200-tp8-c4-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: true - source: - rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd -engine: sglang -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 10 - cuda-graph-max-bs: 10 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - enable-metrics: true -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml deleted file mode 100644 index 98a00e015f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml +++ /dev/null @@ -1,128 +0,0 @@ -schema: 2 -name: agg-gb200-tp8-c8-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: true - source: - rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd -engine: sglang -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 10 - cuda-graph-max-bs: 10 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - enable-metrics: true -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml deleted file mode 100644 index dad9b18d2b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml +++ /dev/null @@ -1,199 +0,0 @@ -schema: 2 -name: disagg-gb200-1p4d-dep8-tp4-c48-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: true - source: - rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd -engine: sglang -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - max-running-requests: 16 - cuda-graph-max-bs: 16 - disable-cuda-graph: true - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 2 - enable-metrics: true - enable-cache-report: true - decode: - nodes: 4 - workers: 4 - gpus: 4 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 16 - cuda-graph-max-bs: 16 - chunked-prefill-size: 64 - context-length: 1048576 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.9 - disaggregation-decode-extra-slots: 0 - enable-metrics: true - enable-cache-report: true -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml deleted file mode 100644 index 13b242d780..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml +++ /dev/null @@ -1,199 +0,0 @@ -schema: 2 -name: disagg-gb200-1p6d-dep8-tp4-c45-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: true - source: - rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd -engine: sglang -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - max-running-requests: 16 - cuda-graph-max-bs: 16 - disable-cuda-graph: true - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 2 - enable-metrics: true - enable-cache-report: true - decode: - nodes: 6 - workers: 6 - gpus: 4 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 16 - cuda-graph-max-bs: 16 - chunked-prefill-size: 64 - context-length: 1048576 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.9 - disaggregation-decode-extra-slots: 0 - enable-metrics: true - enable-cache-report: true -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml deleted file mode 100644 index a586a319ea..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml +++ /dev/null @@ -1,207 +0,0 @@ -schema: 2 -name: disagg-gb200-2p1d-dep8-dep16-c128-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: true - source: - rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd -engine: sglang -roles: - prefill: - nodes: 4 - workers: 2 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 8192 - max-running-requests: 16 - cuda-graph-max-bs: 16 - disable-cuda-graph: true - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 100 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 2 - enable-metrics: true - enable-cache-report: true - decode: - nodes: 4 - workers: 1 - gpus: 16 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 144 - cuda-graph-max-bs: 144 - chunked-prefill-size: 64 - context-length: 1048576 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutedsl - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.85 - disaggregation-decode-extra-slots: 0 - enable-metrics: true - enable-cache-report: true - deepep-config: /configs/deepep_config.json - deepep-mode: low_latency - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - speculative-moe-a2a-backend: deepep - speculative-moe-runner-backend: deep_gemm -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml new file mode 100644 index 0000000000..6411d5880e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml @@ -0,0 +1,247 @@ +# AgentX glm5.2 sglang gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml:override_. + +schema: 2 + +base: + model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 + identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + frameworks: + dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + sglang: nightly-dev-cu13-20260805-211ee642 + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + dynamo: + install: true + source: + rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + engine: sglang + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + max-running-requests: 16 + cuda-graph-max-bs: 16 + disable-cuda-graph: true + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 135 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + enable-metrics: true + enable-cache-report: true + decode: + nodes: 4 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 16 + cuda-graph-max-bs: 16 + chunked-prefill-size: 64 + context-length: 1048576 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.9 + disaggregation-decode-extra-slots: 0 + enable-metrics: true + enable-cache-report: true + health_check: + max_attempts: 1440 + interval_seconds: 10 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + +override_1p6d_tp4_c45: + name: disagg-gb200-1p6d-dep8-tp4-c45-mtp + roles: + decode: + nodes: 6 + workers: 6 + +override_1p4d_tp4_c48: + name: disagg-gb200-1p4d-dep8-tp4-c48-mtp + roles: + decode: + workers: 4 + +override_2p1d_dep16_c128: + name: disagg-gb200-2p1d-dep8-dep16-c128-mtp + roles: + prefill: + nodes: 4 + workers: 2 + args: + max-prefill-tokens: 8192 + hicache-size: 100 + decode: + workers: 1 + gpus: 16 + args: + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 144 + cuda-graph-max-bs: 144 + moe-runner-backend: flashinfer_cutedsl + mem-fraction-static: 0.85 + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + speculative-moe-a2a-backend: deepep + speculative-moe-runner-backend: deep_gemm diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml deleted file mode 100644 index 7ea49d9257..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml +++ /dev/null @@ -1,188 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - args: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - tensor_parallel_size: 4 - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: *server_environment - args: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1, 2, 4, 5] - enable_padding: true - enable_attention_dp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - tokens_per_block: 64 - max_batch_size: 5 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - stream_interval: 20 - tensor_parallel_size: 8 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: head -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - placement: - node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml deleted file mode 100644 index efa7b8fb86..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml +++ /dev/null @@ -1,185 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - args: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - disable_overlap_scheduler: true - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: EQUAL_PROGRESS - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - tensor_parallel_size: 4 - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: *server_environment - args: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1] - enable_padding: true - enable_attention_dp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - tokens_per_block: 64 - max_batch_size: 1 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 1 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - stream_interval: 20 - tensor_parallel_size: 8 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: head -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - placement: - node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml deleted file mode 100644 index 8343835cc1..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml +++ /dev/null @@ -1,188 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - args: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - tensor_parallel_size: 4 - decode: - nodes: 4 - workers: 4 - gpus: 4 - - env: *server_environment - args: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1, 2] - enable_padding: true - enable_attention_dp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - tokens_per_block: 64 - max_batch_size: 2 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - stream_interval: 20 - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: head -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - placement: - node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml deleted file mode 100644 index 70947a0180..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml +++ /dev/null @@ -1,188 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false - -roles: - prefill: - nodes: 3 - workers: 3 - gpus: 4 - env: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - args: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - tensor_parallel_size: 4 - decode: - nodes: 4 - workers: 4 - gpus: 4 - - env: *server_environment - args: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1, 2, 4, 5] - enable_padding: true - enable_attention_dp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - tokens_per_block: 64 - max_batch_size: 5 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - stream_interval: 20 - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: head -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - placement: - node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml deleted file mode 100644 index 60a66c1a5b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml +++ /dev/null @@ -1,189 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-4p1d-dep8-c227-b16-mtp3 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false - -roles: - prefill: - nodes: 4 - workers: 4 - gpus: 4 - env: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - args: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 4 - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: *server_environment - args: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1, 2, 4, 8, 16] - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.9 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 16 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 8 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: head -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - placement: - node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml deleted file mode 100644 index e0cbc51b2e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml +++ /dev/null @@ -1,189 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-5p1d-dep16-c260-b16-mtp3 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false - -roles: - prefill: - nodes: 5 - workers: 5 - gpus: 4 - env: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - args: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 4 - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: *server_environment - args: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1, 2, 4, 8, 16] - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.9 - tokens_per_block: 64 - max_batch_size: 16 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 16 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: head -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - placement: - node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..1cefab7def --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,380 @@ +# AgentX glm5.2 trtllm gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + precision: fp4 + dynamo: + install: true + request_plane: tcp + health_check: + max_attempts: 270 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + engine: + type: trtllm + publish_events_and_metrics: false + roles: + prefill: + gpus: 4 + env: + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_FUSED_DSA_METADATA: "1" + TRTLLM_DSA_INDEXER_BF16: "1" + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + args: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + tensor_parallel_size: 4 + decode: + workers: 1 + env: + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_FUSED_DSA_METADATA: "1" + TRTLLM_DSA_INDEXER_BF16: "1" + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + args: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: head + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + placement: + node: dedicated + +override_1p1d_tp8_c1_b1_mtp5: + name: dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5 + dynamo: + source: + wheel: 1.4.0.dev20260807 + roles: + prefill: + nodes: 1 + workers: 1 + args: + disable_overlap_scheduler: true + scheduler_config: + context_chunking_policy: EQUAL_PROGRESS + decode: + nodes: 2 + gpus: 8 + args: + cuda_graph_config: + batch_sizes: [1] + max_batch_size: 1 + moe_expert_parallel_size: 1 + sparse_attention_config: + use_cute_dsl_topk: true + tensor_parallel_size: 8 + +override_1p1d_tep8_c20_b5_mtp5: + name: dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5 + dynamo: + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + roles: + prefill: + nodes: 1 + workers: 1 + args: + enable_iter_perf_stats: true + decode: + nodes: 2 + gpus: 8 + args: + cuda_graph_config: + batch_sizes: [1, 2, 4, 5] + max_batch_size: 5 + moe_expert_parallel_size: 8 + enable_iter_perf_stats: true + sparse_attention_config: + use_cute_dsl_topk: true + tensor_parallel_size: 8 + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + +override_1p4d_tep4_c30_b2_mtp5: + name: dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5 + dynamo: + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + roles: + prefill: + nodes: 1 + workers: 1 + args: + enable_iter_perf_stats: true + decode: + nodes: 4 + workers: 4 + gpus: 4 + args: + cuda_graph_config: + batch_sizes: [1, 2] + max_batch_size: 2 + moe_expert_parallel_size: 4 + enable_iter_perf_stats: true + sparse_attention_config: + use_cute_dsl_topk: true + tensor_parallel_size: 4 + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + +override_3p4d_tep4_c60_b5_mtp5: + name: dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5 + dynamo: + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + roles: + prefill: + nodes: 3 + workers: 3 + args: + enable_iter_perf_stats: true + decode: + nodes: 4 + workers: 4 + gpus: 4 + args: + cuda_graph_config: + batch_sizes: [1, 2, 4, 5] + max_batch_size: 5 + moe_expert_parallel_size: 4 + enable_iter_perf_stats: true + sparse_attention_config: + use_cute_dsl_topk: true + tensor_parallel_size: 4 + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + +override_4p1d_dep8_c227_b16_mtp3: + name: dynamo-disagg-gb300-4p1d-dep8-c227-b16-mtp3 + dynamo: + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + roles: + prefill: + nodes: 4 + workers: 4 + args: + enable_iter_perf_stats: true + speculative_config: + max_draft_len: 3 + decode: + nodes: 2 + gpus: 8 + args: + cuda_graph_config: + batch_sizes: [1, 2, 4, 8, 16] + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + free_gpu_memory_fraction: 0.9 + host_cache_size: 137438953472 + max_batch_size: 16 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 8 + enable_iter_perf_stats: true + speculative_config: + max_draft_len: 3 + tensor_parallel_size: 8 + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + +override_5p1d_dep16_c260_b16_mtp3: + name: dynamo-disagg-gb300-5p1d-dep16-c260-b16-mtp3 + dynamo: + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + roles: + prefill: + nodes: 5 + workers: 5 + args: + enable_iter_perf_stats: true + speculative_config: + max_draft_len: 3 + decode: + nodes: 4 + gpus: 16 + args: + cuda_graph_config: + batch_sizes: [1, 2, 4, 8, 16] + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + free_gpu_memory_fraction: 0.9 + max_batch_size: 16 + moe_expert_parallel_size: 16 + enable_iter_perf_stats: true + sparse_attention_config: + use_cute_dsl_topk: true + speculative_config: + max_draft_len: 3 + tensor_parallel_size: 16 + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 847d449d74..5e285427fd 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7216,7 +7216,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml:override_c1" - spec-decoding: mtp conc-list: [4] kv-offloading: dram @@ -7228,7 +7228,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml:override_c4" - spec-decoding: mtp conc-list: [8] kv-offloading: dram @@ -7240,7 +7240,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml:override_c8" glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-20260910-00840301 @@ -7268,7 +7268,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml:override_1p4d_tp4_c48" decode: num-worker: 4 tp: 4 @@ -7285,7 +7285,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml:override_1p1d_c64" decode: num-worker: 1 tp: 8 @@ -7382,7 +7382,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 8 dp-attn: true additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml:override_1p6d_tp4_c45 decode: num-worker: 6 tp: 4 @@ -7400,7 +7400,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 8 dp-attn: true additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml:override_1p4d_tp4_c48 decode: num-worker: 4 tp: 4 @@ -7418,7 +7418,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 8 dp-attn: true additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml:override_2p1d_dep16_c128 decode: num-worker: 1 tp: 16 @@ -7456,7 +7456,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml:override_c2 - spec-decoding: mtp kv-offloading: dram kv-offload-backend: @@ -7470,7 +7470,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml:override_c4 - spec-decoding: mtp kv-offloading: dram kv-offload-backend: @@ -7484,7 +7484,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml:override_c8 glm5.2-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 model: nvidia/GLM-5.2-NVFP4 @@ -7639,7 +7639,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tp8_c1_b1_mtp5" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -7656,7 +7656,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p4d_tep4_c30_b2_mtp5" - "SLURM_PARTITION=batch_1" decode: num-worker: 4 @@ -7673,7 +7673,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tep8_c20_b5_mtp5" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -7690,7 +7690,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_3p4d_tep4_c60_b5_mtp5" - "SLURM_PARTITION=batch_1" decode: num-worker: 4 @@ -7707,7 +7707,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_5p1d_dep16_c260_b16_mtp3" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -7724,7 +7724,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_4p1d_dep8_c227_b16_mtp3" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 From 658f8c038ac360d228af9e73ad4457de8640d9da Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:37:32 -0500 Subject: [PATCH 074/105] refactor(agentx): consolidate Kimi-K3 GB200 AgentX recipes into override variants Each per-configuration recipe becomes an `override_` block over a shared `base` in one `*-variants.yaml` per master-config entry, and the master entries select it with `CONFIG_FILE=...:override_`. Every selected variant resolves, through the pinned srtctl, to exactly the recipe it replaces, including its original `name`. Power recipes with top-level telemetry stay standalone because launchers detect them as text. --- .../agentx/agg-dep16-vllm-simple-offload.yaml | 163 ------------ .../vllm/gb200-fp4/agentx/agg-dep16.yaml | 160 ------------ .../gb200-fp4/agentx/agg-tep16-balanced.yaml | 151 ----------- .../gb200-fp4/agentx/agg-tp16-latency.yaml | 149 ----------- .../vllm/gb200-fp4/agentx/agg-variants.yaml | 238 ++++++++++++++++++ configs/nvidia-master.yaml | 8 +- 6 files changed, 242 insertions(+), 627 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml deleted file mode 100644 index 9d8b740e1f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml +++ /dev/null @@ -1,163 +0,0 @@ -schema: 2 -name: "kimi-k3-vllm-agg-gb200-dep16-throughput-vllm-simple-offload-agentic" - -# High-concurrency host-DRAM KV-offload variant of the official throughput- -# oriented multi_node_dep profile. TP4 x DP4 gives EP16 across four four-GPU -# GB200 nodes. Each TP rank receives a 128 GiB CPU KV pool (512 GiB per node). -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_dep - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - install: true - - source: - rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "12:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - args: - # Use vLLM's Kimi K3 parser so OpenAI responses expose structured - # tool_calls instead of raw XTML in message.content. - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 1 - kv-cache-block-size: 64 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - NVIDIA_GDRCOPY: "1" - PYTHONHASHSEED: "42" - PYTORCH_ALLOC_CONF: "expandable_segments:True" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-dep16-offload-{job_id}" - args: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - # FlashInfer's larger TP4 MoE representation leaves too little transient - # HBM for fastsafetensors' GPU-staging path when DSpark is loaded. - load-format: "safetensors" - safetensors-load-strategy: "lazy" - kv-cache-dtype: "fp8" - attention-backend: "FLASHINFER_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - # DeepGEMM mega-MoE grid barriers time out under DSpark MRV2 with the - # one-sided all-to-all worker; use the supported Kimi K3 FlashInfer path. - moe-backend: "flashinfer_trtllm" - kda-prefill-backend: "flashkda" - kernel-config: '{"enable_cutedsl_warmup":true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.94 - # The largest offload point is c384 / DP4 = 96 sequences per engine. - # Capture even sequence counts: all configured DP4 steady-state batch - # sizes are exact hits, while odd loads pad by at most one sequence. - max-num-seqs: 96 - max-num-batched-tokens: 16384 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96,102,108,114,120,126,132,138,144,150,156,162,168,174,180,186,192,198,204,210,216,222,228,234,240,246,252,258,264,270,276,282,288]}' - block-size: 64 - language-model-only: true - disable-custom-all-reduce: true - enable-prefix-caching: true - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' - scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" - dyn-tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - dyn-reasoning-parser: "kimi_k3" - no-enable-flashinfer-autotune: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AGENTIC_WARMUP_GRACE_PERIOD: "3600" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml deleted file mode 100644 index 8bfda2949e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml +++ /dev/null @@ -1,160 +0,0 @@ -schema: 2 -name: "kimi-k3-vllm-agg-gb200-dep16-throughput-agentic" - -# Day-0 GB200 translation of the official throughput-oriented multi_node_dep -# profile. TP4 x DP4 gives EP16 across four four-GPU GB200 nodes. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_dep - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - install: true - - source: - rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "12:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - args: - # Use vLLM's Kimi K3 parser so OpenAI responses expose structured - # tool_calls instead of raw XTML in message.content. - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 1 - kv-cache-block-size: 64 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - NVIDIA_GDRCOPY: "1" - PYTORCH_ALLOC_CONF: "expandable_segments:True" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-dep16-{job_id}" - args: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - # FlashInfer's larger TP4 MoE representation leaves too little transient - # HBM for fastsafetensors' GPU-staging path when DSpark is loaded. - load-format: "safetensors" - safetensors-load-strategy: "lazy" - kv-cache-dtype: "fp8" - attention-backend: "FLASHINFER_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - # DeepGEMM mega-MoE grid barriers time out under DSpark MRV2 with the - # one-sided all-to-all worker; use the supported Kimi K3 FlashInfer path. - moe-backend: "flashinfer_trtllm" - kda-prefill-backend: "flashkda" - kernel-config: '{"enable_cutedsl_warmup":true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.94 - # The largest regular DEP point is c256 / DP4 = 64 sequences per engine. - # Capturing 128 sequence slots consumes 10.9 GiB and leaves too little - # runtime workspace for FlashInfer's MXFP4 MoE kernel. - max-num-seqs: 64 - max-num-batched-tokens: 16384 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96,99,102,105,108,111,114,117,120,123,126,129,132,135,138,141,144,147,150,153,156,159,162,165,168,171,174,177,180,183,186,189,192]}' - block-size: 64 - language-model-only: true - disable-custom-all-reduce: true - enable-prefix-caching: true - scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" - dyn-tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - dyn-reasoning-parser: "kimi_k3" - no-enable-flashinfer-autotune: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AGENTIC_WARMUP_GRACE_PERIOD: "3600" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml deleted file mode 100644 index 58bae20c35..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml +++ /dev/null @@ -1,151 +0,0 @@ -schema: 2 -name: "kimi-k3-vllm-agg-gb200-tep16-balanced-agentic" - -# Day-0 GB200 translation of the official balanced multi_node_tep profile. -# Dense layers and MoE experts are sharded across 16 GPUs on four GB200 nodes -# with the official FP8 KV cache. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tep - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - install: true - - source: - rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - args: - # Use vLLM's Kimi K3 parser so OpenAI responses expose structured - # tool_calls instead of raw XTML in message.content. - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 1 - kv-cache-block-size: 64 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - NVIDIA_GDRCOPY: "1" - PYTORCH_ALLOC_CONF: "expandable_segments:True" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-tep16-{job_id}" - args: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 - pipeline-parallel-size: 1 - enable-expert-parallel: true - trust-remote-code: true - load-format: "fastsafetensors" - safetensors-load-strategy: "lazy" - kv-cache-dtype: "fp8" - attention-backend: "FLASHINFER_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - moe-backend: "flashinfer_trtllm" - kda-prefill-backend: "flashkda" - kernel-config: '{"enable_cutedsl_warmup":true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.92 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96],"pass_config":{"fuse_allreduce_rms":false}}' - block-size: 64 - language-model-only: true - disable-custom-all-reduce: true - enable-prefix-caching: true - scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" - dyn-tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - dyn-reasoning-parser: "kimi_k3" - no-enable-flashinfer-autotune: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml deleted file mode 100644 index 9c597e1701..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml +++ /dev/null @@ -1,149 +0,0 @@ -schema: 2 -name: "kimi-k3-vllm-agg-gb200-tp16-latency-agentic" - -# Day-0 GB200 translation of the official latency-oriented multi_node_tp -# profile. TP16 spans four GB200 nodes and uses the official FP8 KV cache. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tp - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - install: true - - source: - rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - args: - # Use vLLM's Kimi K3 parser so OpenAI responses expose structured - # tool_calls instead of raw XTML in message.content. - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 1 - kv-cache-block-size: 64 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - NVIDIA_GDRCOPY: "1" - PYTORCH_ALLOC_CONF: "expandable_segments:True" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-tp16-{job_id}" - args: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 - pipeline-parallel-size: 1 - trust-remote-code: true - load-format: "fastsafetensors" - safetensors-load-strategy: "lazy" - kv-cache-dtype: "fp8" - attention-backend: "FLASHINFER_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - moe-backend: "flashinfer_trtllm" - kda-prefill-backend: "flashkda" - kernel-config: '{"enable_cutedsl_warmup":true}' - gpu-memory-utilization: 0.92 - max-num-seqs: 8 - max-num-batched-tokens: 8192 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24],"pass_config":{"fuse_allreduce_rms":false}}' - block-size: 64 - language-model-only: true - disable-custom-all-reduce: true - enable-prefix-caching: true - scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" - dyn-tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - dyn-reasoning-parser: "kimi_k3" - no-enable-flashinfer-autotune: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..4afb5e8a62 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml @@ -0,0 +1,238 @@ +# AgentX kimik3 vllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "kimi-k3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0" + dynamo: + install: true + source: + rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + setup_script: kimik3-dspark-config-compat.sh + environment: + ETCD_LEASE_TTL: "7200" + health_check: + max_attempts: 2160 + interval_seconds: 10 + resources: + gpu_type: "gb200" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + enable_multiple_frontends: false + args: + # Use vLLM's Kimi K3 parser so OpenAI responses expose structured + # tool_calls instead of raw XTML in message.content. + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + enable-auto-tool-choice: true + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-min-initial-workers: 1 + kv-cache-block-size: 64 + engine: + type: vllm + connector: + roles: + agg: + nodes: 4 + workers: 1 + gpus: 16 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + NVIDIA_GDRCOPY: "1" + PYTORCH_ALLOC_CONF: "expandable_segments:True" + args: + served-model-name: "moonshotai/Kimi-K3" + pipeline-parallel-size: 1 + trust-remote-code: true + safetensors-load-strategy: "lazy" + kv-cache-dtype: "fp8" + attention-backend: "FLASHINFER_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # DeepGEMM mega-MoE grid barriers time out under DSpark MRV2 with the + # one-sided all-to-all worker; use the supported Kimi K3 FlashInfer path. + moe-backend: "flashinfer_trtllm" + kda-prefill-backend: "flashkda" + kernel-config: '{"enable_cutedsl_warmup":true}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + block-size: 64 + language-model-only: true + disable-custom-all-reduce: true + enable-prefix-caching: true + scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" + dyn-tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + dyn-reasoning-parser: "kimi_k3" + no-enable-flashinfer-autotune: true + sbatch_directives: + cpus-per-task: "144" + mem: "0" + srun_options: + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +# Day-0 GB200 translation of the official latency-oriented multi_node_tp +# profile. TP16 spans four GB200 nodes and uses the official FP8 KV cache. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tp +override_tp16_latency: + name: "kimi-k3-vllm-agg-gb200-tp16-latency-agentic" + slurm: + time_limit: "8:00:00" + roles: + agg: + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-tp16-{job_id}" + args: + tensor-parallel-size: 16 + load-format: "fastsafetensors" + gpu-memory-utilization: 0.92 + max-num-seqs: 8 + max-num-batched-tokens: 8192 + compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24],"pass_config":{"fuse_allreduce_rms":false}}' + benchmark: + env: + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + +# Day-0 GB200 translation of the official balanced multi_node_tep profile. +# Dense layers and MoE experts are sharded across 16 GPUs on four GB200 nodes +# with the official FP8 KV cache. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tep +override_tep16_balanced: + name: "kimi-k3-vllm-agg-gb200-tep16-balanced-agentic" + slurm: + time_limit: "8:00:00" + roles: + agg: + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-tep16-{job_id}" + args: + tensor-parallel-size: 16 + enable-expert-parallel: true + load-format: "fastsafetensors" + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.92 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96],"pass_config":{"fuse_allreduce_rms":false}}' + +# Day-0 GB200 translation of the official throughput-oriented multi_node_dep +# profile. TP4 x DP4 gives EP16 across four four-GPU GB200 nodes. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_dep +override_dep16: + name: "kimi-k3-vllm-agg-gb200-dep16-throughput-agentic" + slurm: + time_limit: "12:00:00" + roles: + agg: + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-dep16-{job_id}" + args: + tensor-parallel-size: 4 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + # FlashInfer's larger TP4 MoE representation leaves too little transient + # HBM for fastsafetensors' GPU-staging path when DSpark is loaded. + load-format: "safetensors" + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.94 + # The largest regular DEP point is c256 / DP4 = 64 sequences per engine. + # Capturing 128 sequence slots consumes 10.9 GiB and leaves too little + # runtime workspace for FlashInfer's MXFP4 MoE kernel. + max-num-seqs: 64 + max-num-batched-tokens: 16384 + compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96,99,102,105,108,111,114,117,120,123,126,129,132,135,138,141,144,147,150,153,156,159,162,165,168,171,174,177,180,183,186,189,192]}' + benchmark: + env: + AGENTIC_WARMUP_GRACE_PERIOD: "3600" + +# High-concurrency host-DRAM KV-offload variant of the official throughput- +# oriented multi_node_dep profile. TP4 x DP4 gives EP16 across four four-GPU +# GB200 nodes. Each TP rank receives a 128 GiB CPU KV pool (512 GiB per node). +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_dep +override_dep16_vllm_simple_offload: + name: "kimi-k3-vllm-agg-gb200-dep16-throughput-vllm-simple-offload-agentic" + slurm: + time_limit: "12:00:00" + roles: + agg: + env: + PYTHONHASHSEED: "42" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-dep16-offload-{job_id}" + args: + tensor-parallel-size: 4 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + # FlashInfer's larger TP4 MoE representation leaves too little transient + # HBM for fastsafetensors' GPU-staging path when DSpark is loaded. + load-format: "safetensors" + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.94 + # The largest offload point is c384 / DP4 = 96 sequences per engine. + # Capture even sequence counts: all configured DP4 steady-state batch + # sizes are exact hits, while odd loads pad by at most one sequence. + max-num-seqs: 96 + max-num-batched-tokens: 16384 + compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96,102,108,114,120,126,132,138,144,150,156,162,168,174,180,186,192,198,204,210,216,222,228,234,240,246,252,258,264,270,276,282,288]}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' + benchmark: + env: + AGENTIC_WARMUP_GRACE_PERIOD: "3600" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 5e285427fd..1e03899e43 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6385,7 +6385,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml:override_tp16_latency" # Balanced: multi_node_tep strategy, TEP16 across four GB200 nodes. # https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tep - spec-decoding: mtp @@ -6397,7 +6397,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: ep: 16 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml:override_tep16_balanced" # Throughput oriented: official multi_node_dep strategy, DEP16 across # four GB200 nodes (TP4 x DP4 = EP16, one local DP rank per node). # The recipe allows up to 3600s for full-context saturation warmup to drain. @@ -6411,7 +6411,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: ep: 16 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml:override_dep16" # High-concurrency DEP16 with vLLM Simple CPU KV offloading. c384 # exercises 384 of the 393 AgentX trajectories and remains below the # aggregate max-num-seqs capacity of 512 (128 per DP rank). @@ -6426,7 +6426,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: ep: 16 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml:override_dep16_vllm_simple_offload" # Kimi-K3 GB200 TP16/DCP16 profiles using Mooncake DRAM offload. kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-agg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef From 9b233f15269004a922e55a696eb4a33d8dd19e10 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:37:32 -0500 Subject: [PATCH 075/105] refactor(agentx): consolidate MiniMax-M3 multi-node AgentX recipes into override variants Each per-configuration recipe becomes an `override_` block over a shared `base` in one `*-variants.yaml` per master-config entry, and the master entries select it with `CONFIG_FILE=...:override_`. Every selected variant resolves, through the pinned srtctl, to exactly the recipe it replaces, including its original `name`. Power recipes with top-level telemetry stay standalone because launchers detect them as text. --- .../agentx/agg-tp4-c10-b10-eagle3.yaml | 152 ------- .../agentx/agg-tp4-c15-b15-eagle3.yaml | 157 -------- .../agentx/agg-tp4-c20-b20-eagle3.yaml | 162 -------- .../agentx/agg-tp4-c25-b25-eagle3.yaml | 162 -------- .../agentx/agg-tp4-c30-b30-eagle3.yaml | 162 -------- .../agentx/agg-tp4-c40-b40-eagle3.yaml | 166 -------- .../agentx/agg-tp4-c5-b5-eagle3.yaml | 147 ------- .../trtllm/gb200-fp4/agentx/agg-variants.yaml | 372 ++++++++++++++++++ .../agentx/agg-nightly-native-variants.yaml | 129 ++++++ .../agentx/agg-tp4-nightly-native.yaml | 111 ------ .../agg-tp4-vllm-simple-nightly-native.yaml | 108 ----- .../agentx/agg-tp8-nightly-native.yaml | 111 ------ .../agentx/disagg-1p1d-tp4-tp4-c24.yaml | 173 -------- .../agentx/disagg-1p1d-tp4-tp8-c1.yaml | 180 --------- .../agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml | 173 -------- .../gb200-fp4/agentx/disagg-variants.yaml | 218 ++++++++++ .../agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml | 179 --------- .../agentx/disagg-1p1d-tep4-tp4-c1.yaml | 179 --------- .../disagg-1p1d-tp2-tp4-c20-c24-eval.yaml | 179 --------- .../agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml | 179 --------- .../agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml | 182 --------- .../agentx/disagg-1p3d-dep4-tp4-c24.yaml | 182 --------- .../agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml | 179 --------- .../agentx/disagg-1p3d-tp2-tp2-c48.yaml | 179 --------- .../agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml | 179 --------- .../agentx/disagg-2p5d-tp2-tp2-c120.yaml | 179 --------- .../gb300-fp4/agentx/disagg-variants.yaml | 364 +++++++++++++++++ configs/nvidia-master.yaml | 46 +-- 28 files changed, 1106 insertions(+), 3783 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml deleted file mode 100644 index 14d4fcbe7f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml +++ /dev/null @@ -1,152 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=10 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# AgentX acceptance is selected from the committed golden curve at submission. -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -schema: 2 -name: dynamo-agg-gb200-tp4-c10-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 -engine: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - publish_events_and_metrics: false -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - args: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 10 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '10' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml deleted file mode 100644 index f18cbf28d0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml +++ /dev/null @@ -1,157 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=15 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# AgentX acceptance is selected from the committed golden curve at submission. -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -schema: 2 -name: dynamo-agg-gb200-tp4-c15-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 -engine: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - publish_events_and_metrics: false -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - args: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 15 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 13 - - 14 - - 15 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '15' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml deleted file mode 100644 index 4bbe9c29e0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml +++ /dev/null @@ -1,162 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=20 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# AgentX acceptance is selected from the committed golden curve at submission. -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -schema: 2 -name: dynamo-agg-gb200-tp4-c20-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 -engine: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - publish_events_and_metrics: false -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - args: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 20 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 13 - - 14 - - 15 - - 16 - - 17 - - 18 - - 19 - - 20 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '20' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml deleted file mode 100644 index d0c79d1e41..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml +++ /dev/null @@ -1,162 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=25 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# AgentX acceptance is selected from the committed golden curve at submission. -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -schema: 2 -name: dynamo-agg-gb200-tp4-c25-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 -engine: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - publish_events_and_metrics: false -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - args: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 25 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 13 - - 14 - - 15 - - 17 - - 19 - - 21 - - 23 - - 25 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '25' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml deleted file mode 100644 index 5cecb444a3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml +++ /dev/null @@ -1,162 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=30 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# AgentX acceptance is selected from the committed golden curve at submission. -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -schema: 2 -name: dynamo-agg-gb200-tp4-c30-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 -engine: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - publish_events_and_metrics: false -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - args: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 30 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 14 - - 16 - - 18 - - 20 - - 22 - - 24 - - 27 - - 30 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '30' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml deleted file mode 100644 index bf0920ce90..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml +++ /dev/null @@ -1,166 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=40 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# AgentX acceptance is selected from the committed golden curve at submission. -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -schema: 2 -name: dynamo-agg-gb200-tp4-c40-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 -engine: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - publish_events_and_metrics: false -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - args: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 40 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 14 - - 16 - - 18 - - 20 - - 22 - - 24 - - 26 - - 28 - - 30 - - 32 - - 34 - - 36 - - 38 - - 40 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '40' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml deleted file mode 100644 index 1eaa2bce7b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml +++ /dev/null @@ -1,147 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=5 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# AgentX acceptance is selected from the committed golden curve at submission. -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -schema: 2 -name: dynamo-agg-gb200-tp4-c5-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 -engine: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - publish_events_and_metrics: false -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - args: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 5 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '5' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..6ac78e12b7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml @@ -0,0 +1,372 @@ +# AgentX minimaxm3 trtllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: minimax-m3-nvfp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 + precision: fp4 + dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + request_plane: tcp + health_check: + max_attempts: 270 + interval_seconds: 10 + resources: + gpu_type: gb200 + gpus_per_node: 4 + engine: + type: trtllm + numa_memory_bind: true + served_model_name: nvidia/MiniMax-M3-NVFP4 + publish_events_and_metrics: false + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + HF_HUB_CACHE: /hf_hub_cache + TQDM_DISABLE: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + PYTHONNOUSERSITE: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + args: + max_seq_len: 1048576 + max_num_tokens: 16384 + cuda_graph_config: + enable_padding: true + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: + - 1 + - 512 + - 1024 + - 2048 + enable_userbuffers: true + max_num_streams: 3 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + free_gpu_memory_fraction: 0.94 + enable_block_reuse: true + block_reuse_policy: per_conversation + tokens_per_block: 128 + use_kv_cache_manager_v2: true + dtype: fp8 + event_buffer_max_size: 0 + host_cache_size: 137438953472 + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + enable_chunked_prefill: true + enable_autotuner: true + trust_remote_code: true + stream_interval: 20 + print_iter_log: true + num_postprocess_workers: 8 + enable_attention_dp: false + tensor_parallel_size: 4 + frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + KV_OFFLOAD_BACKEND: native + KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' + TOTAL_CPU_DRAM_GB: '550' + MODEL: nvidia/MiniMax-M3-NVFP4 + MODEL_PREFIX: minimaxm3 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + DURATION: '3600' + +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=5 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# AgentX acceptance is selected from the committed golden curve at submission. +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +override_c5_b5: + name: dynamo-agg-gb200-tp4-c5-b1-eagle3 + roles: + agg: + args: + max_batch_size: 5 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + benchmark: + env: + CONC: '5' + +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=10 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# AgentX acceptance is selected from the committed golden curve at submission. +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +override_c10_b10: + name: dynamo-agg-gb200-tp4-c10-b1-eagle3 + roles: + agg: + args: + max_batch_size: 10 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + benchmark: + env: + CONC: '10' + +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=15 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# AgentX acceptance is selected from the committed golden curve at submission. +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +override_c15_b15: + name: dynamo-agg-gb200-tp4-c15-b1-eagle3 + roles: + agg: + args: + max_batch_size: 15 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + benchmark: + env: + CONC: '15' + +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=20 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# AgentX acceptance is selected from the committed golden curve at submission. +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +override_c20_b20: + name: dynamo-agg-gb200-tp4-c20-b1-eagle3 + roles: + agg: + args: + max_batch_size: 20 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + benchmark: + env: + CONC: '20' + +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=25 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# AgentX acceptance is selected from the committed golden curve at submission. +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +override_c25_b25: + name: dynamo-agg-gb200-tp4-c25-b1-eagle3 + roles: + agg: + args: + max_batch_size: 25 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 17 + - 19 + - 21 + - 23 + - 25 + benchmark: + env: + CONC: '25' + +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=30 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# AgentX acceptance is selected from the committed golden curve at submission. +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +override_c30_b30: + name: dynamo-agg-gb200-tp4-c30-b1-eagle3 + roles: + agg: + args: + max_batch_size: 30 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 27 + - 30 + benchmark: + env: + CONC: '30' + +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=40 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# AgentX acceptance is selected from the committed golden curve at submission. +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +override_c40_b40: + name: dynamo-agg-gb200-tp4-c40-b1-eagle3 + roles: + agg: + args: + max_batch_size: 40 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 26 + - 28 + - 30 + - 32 + - 34 + - 36 + - 38 + - 40 + benchmark: + env: + CONC: '40' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml new file mode 100644 index 0000000000..bd8c7bc9b6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml @@ -0,0 +1,129 @@ +# AgentX minimaxm3 vllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "minimax-m3-nvfp4" + container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" + precision: "fp4" + identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7"} + frameworks: {dynamo: "1.5.0.dev20260908"} + dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} + environment: {ETCD_LEASE_TTL: "7200"} + slurm: {time_limit: "12:00:00"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: "gb200" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + enable_multiple_frontends: false + env: {DYN_TCP_REQUEST_TIMEOUT: "60"} + args: + trust-remote-code: true + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 + engine: + type: vllm + connector: + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + kv_events: true + sbatch_directives: {cpus-per-task: "144", mem: "0"} + srun_options: {container-remap-root: ""} + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +override_tp4: + name: "minimax-m3-vllm-agg-gb200-tp4-agentic-nightly-native" + +override_tp4_vllm_simple: + name: "minimax-m3-vllm-agg-gb200-tp4-vllm-simple-agentic-nightly-native" + roles: + agg: + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + args: + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":777389080576,"cpu_bytes_to_use_per_rank":194347270144,"lazy_offload":true}}' + +override_tp8: + name: "minimax-m3-vllm-agg-gb200-tp8-agentic-nightly-native" + roles: + agg: + nodes: 2 + gpus: 8 + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + args: + tensor-parallel-size: 8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml deleted file mode 100644 index 6e8399d239..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml +++ /dev/null @@ -1,111 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-agg-gb200-tp4-agentic-nightly-native" - -model: - path: "minimax-m3-nvfp4" - container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" - precision: "fp4" - -identity: - model: {repo: "nvidia/MiniMax-M3-NVFP4"} - container: {image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7"} - frameworks: {dynamo: "1.5.0.dev20260908"} - -dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} -environment: {ETCD_LEASE_TTL: "7200"} - -slurm: {time_limit: "12:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: {DYN_TCP_REQUEST_TIMEOUT: "60"} - args: - trust-remote-code: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - - env: - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - - kv_events: true -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml deleted file mode 100644 index a36e5ff622..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml +++ /dev/null @@ -1,108 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-agg-gb200-tp4-vllm-simple-agentic-nightly-native" - -model: - path: "minimax-m3-nvfp4" - container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" - precision: "fp4" - -identity: - model: {repo: "nvidia/MiniMax-M3-NVFP4"} - container: {image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7"} - frameworks: {dynamo: "1.5.0.dev20260908"} - -dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} -environment: {ETCD_LEASE_TTL: "7200"} -slurm: {time_limit: "12:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} -resources: {gpu_type: "gb200", gpus_per_node: 4} -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: {DYN_TCP_REQUEST_TIMEOUT: "60"} - args: - trust-remote-code: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_SIMPLE_KV_OFFLOAD: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":777389080576,"cpu_bytes_to_use_per_rank":194347270144,"lazy_offload":true}}' - stream-interval: 20 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - - kv_events: true -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml deleted file mode 100644 index 822ca199c8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml +++ /dev/null @@ -1,111 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-agg-gb200-tp8-agentic-nightly-native" - -model: - path: "minimax-m3-nvfp4" - container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" - precision: "fp4" - -identity: - model: {repo: "nvidia/MiniMax-M3-NVFP4"} - container: {image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7"} - frameworks: {dynamo: "1.5.0.dev20260908"} - -dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} -environment: {ETCD_LEASE_TTL: "7200"} - -slurm: {time_limit: "12:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: {DYN_TCP_REQUEST_TIMEOUT: "60"} - args: - trust-remote-code: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - - env: - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' - stream-interval: 20 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - - kv_events: true -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml deleted file mode 100644 index a9f4cbc3eb..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml +++ /dev/null @@ -1,173 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb200-1p1d-tp4-tp4-c24-agentic" - -model: - path: "minimax-m3-nvfp4" - container: &container "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6" - precision: "fp4" - -identity: - model: {repo: "nvidia/MiniMax-M3-NVFP4"} - container: {image: *container} - frameworks: {dynamo: "1.5.0.dev20260819"} - -dynamo: {install: true, source: {wheel: "1.5.0.dev20260819"}} -environment: {PYTHONHASHSEED: "0"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 3600 - env: - DYN_LOG: "info" - DYN_TCP_CONNECT_TIMEOUT: "120" - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE: "1" - -engine: - type: vllm - connector: - dp_launch_mode: per_gpu -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &worker_environment - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MEMTYPE_CACHE: "n" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - UCX_TLS: "tcp,cuda_ipc,cuda_copy" - WITH_NVIDIA_PEERMEM: "0" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "1" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: &numa_nodes [0, 0, 1, 1] - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - nodes: 1 - workers: 1 - gpus: 4 - - env: *worker_environment - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: *numa_nodes - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AGENTIC_WARMUP_GRACE_PERIOD: "1800" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "3600" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml deleted file mode 100644 index 2aa021e119..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml +++ /dev/null @@ -1,180 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb200-1p1d-tp4-tp8-c1-agentic" - -model: - path: "minimax-m3-nvfp4" - container: &container "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6" - precision: "fp4" - -identity: - model: {repo: "nvidia/MiniMax-M3-NVFP4"} - container: {image: *container} - frameworks: {dynamo: "1.5.0.dev20260819"} - -dynamo: {install: true, source: {wheel: "1.5.0.dev20260819"}} -environment: {PYTHONHASHSEED: "0"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 3600 - env: - DYN_LOG: "info" - DYN_TCP_CONNECT_TIMEOUT: "120" - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE: "1" - -engine: - type: vllm - connector: - dp_launch_mode: per_gpu -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &worker_environment - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MEMTYPE_CACHE: "n" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - UCX_TLS: "tcp,cuda_ipc,cuda_copy" - WITH_NVIDIA_PEERMEM: "0" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "1" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: &numa_nodes [0, 0, 1, 1] - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "1" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 8 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-num-seqs: 1 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: *numa_nodes - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AGENTIC_WARMUP_GRACE_PERIOD: "1800" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "3600" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml deleted file mode 100644 index dd50b12a6f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml +++ /dev/null @@ -1,173 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb200-1p2d-tp4-tp4-c8-c16-agentic" - -model: - path: "minimax-m3-nvfp4" - container: &container "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6" - precision: "fp4" - -identity: - model: {repo: "nvidia/MiniMax-M3-NVFP4"} - container: {image: *container} - frameworks: {dynamo: "1.5.0.dev20260819"} - -dynamo: {install: true, source: {wheel: "1.5.0.dev20260819"}} -environment: {PYTHONHASHSEED: "0"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 3600 - env: - DYN_LOG: "info" - DYN_TCP_CONNECT_TIMEOUT: "120" - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE: "1" - -engine: - type: vllm - connector: - dp_launch_mode: per_gpu -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &worker_environment - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MEMTYPE_CACHE: "n" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - UCX_TLS: "tcp,cuda_ipc,cuda_copy" - WITH_NVIDIA_PEERMEM: "0" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "1" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: &numa_nodes [0, 0, 1, 1] - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - nodes: 2 - workers: 2 - gpus: 4 - - env: *worker_environment - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: *numa_nodes - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AGENTIC_WARMUP_GRACE_PERIOD: "1800" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "3600" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..8899d709fc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,218 @@ +# AgentX minimaxm3 vllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "minimax-m3-nvfp4" + container: "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6" + precision: "fp4" + identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6"} + frameworks: {dynamo: "1.5.0.dev20260819"} + dynamo: {install: true, source: {wheel: "1.5.0.dev20260819"}} + environment: {PYTHONHASHSEED: "0"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: "gb200" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "minimax_m3" + reasoning-parser: "minimax_m3" + enable-auto-tool-choice: true + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 3600 + env: + DYN_LOG: "info" + DYN_TCP_CONNECT_TIMEOUT: "120" + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE: "1" + engine: + type: vllm + connector: + dp_launch_mode: per_gpu + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + UCX_TLS: "tcp,cuda_ipc,cuda_copy" + WITH_NVIDIA_PEERMEM: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "1" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 128 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + decode: + nodes: 2 + workers: 1 + gpus: 4 + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "1" + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + UCX_TLS: "tcp,cuda_ipc,cuda_copy" + WITH_NVIDIA_PEERMEM: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 128 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + sbatch_directives: {cpus-per-task: "144", mem: "0"} + srun_options: {container-remap-root: ""} + benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "3600" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +override_1p1d_tp8_c1: + name: "minimax-m3-vllm-disagg-gb200-1p1d-tp4-tp8-c1-agentic" + roles: + decode: + gpus: 8 + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + args: + tensor-parallel-size: 8 + max-num-seqs: 1 + +override_1p2d_c8_c16: + name: "minimax-m3-vllm-disagg-gb200-1p2d-tp4-tp4-c8-c16-agentic" + roles: + decode: + workers: 2 + +override_1p1d_c24: + name: "minimax-m3-vllm-disagg-gb200-1p1d-tp4-tp4-c24-agentic" + roles: + decode: + nodes: 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml deleted file mode 100644 index 02b9c15d97..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-eval-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 1 - workers: 1 - gpus: 4 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml deleted file mode 100644 index 3cac730921..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 1 - workers: 1 - gpus: 4 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml deleted file mode 100644 index 8faf2a5448..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-eval-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 1 - workers: 1 - gpus: 4 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml deleted file mode 100644 index cf4b418c39..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 1 - workers: 1 - gpus: 4 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml deleted file mode 100644 index 400798973b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml +++ /dev/null @@ -1,182 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-eval-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 3 - workers: 3 - gpus: 4 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml deleted file mode 100644 index e541182719..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml +++ /dev/null @@ -1,182 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 3 - workers: 3 - gpus: 4 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml deleted file mode 100644 index 6cf7e59d1e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-eval-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: true -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 3 - workers: 3 - gpus: 2 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml deleted file mode 100644 index cc193dd0c8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: true -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 3 - workers: 3 - gpus: 2 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml deleted file mode 100644 index 66e6e7135d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-eval-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: true -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 2 - workers: 2 - gpus: 2 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 5 - workers: 5 - gpus: 2 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml deleted file mode 100644 index a361fd07c9..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: true -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 2 - workers: 2 - gpus: 2 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 5 - workers: 5 - gpus: 2 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..eefda7020c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,364 @@ +# AgentX minimaxm3 vllm gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + dynamo: + install: true + source: + pypi: "1.4.0.dev20260730" + health_check: + max_attempts: 2160 + interval_seconds: 10 + resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + environment: + PYTHONHASHSEED: "0" + frontend: + type: dynamo + enable_multiple_frontends: true + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + engine: + type: vllm + connector: + dp_launch_mode: per_node + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + gpus: 4 + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + sbatch_directives: + cpus-per-task: "72" + mem: "0" + srun_options: + container-remap-root: "" + benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +override_1p1d_tep4_tp4_c1: + name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-agentic" + frontend: + enable_multiple_frontends: false + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + decode: + nodes: 1 + workers: 1 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + +override_1p1d_tep4_tp4_c1_eval: + name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-eval-agentic" + frontend: + enable_multiple_frontends: false + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + decode: + nodes: 1 + workers: 1 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + +override_1p1d_tp2_tp4_c20_c24: + name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-agentic" + frontend: + num_additional_frontends: 1 + roles: + prefill: + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + decode: + nodes: 1 + workers: 1 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + +override_1p1d_tp2_tp4_c20_c24_eval: + name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-eval-agentic" + frontend: + num_additional_frontends: 1 + roles: + prefill: + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + decode: + nodes: 1 + workers: 1 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + +override_1p3d_dep4_tp4_c24: + name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-agentic" + frontend: + num_additional_frontends: 1 + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + decode: + nodes: 3 + workers: 3 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + +override_1p3d_dep4_tp4_c24_eval: + name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-eval-agentic" + frontend: + num_additional_frontends: 1 + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + decode: + nodes: 3 + workers: 3 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + +override_1p3d_tp2_tp2_c48: + name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-agentic" + resources: + spread_workers: true + frontend: + num_additional_frontends: 4 + roles: + prefill: + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + decode: + nodes: 3 + workers: 3 + gpus: 2 + args: + tensor-parallel-size: 2 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + +override_1p3d_tp2_tp2_c48_eval: + name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-eval-agentic" + resources: + spread_workers: true + frontend: + num_additional_frontends: 4 + roles: + prefill: + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + decode: + nodes: 3 + workers: 3 + gpus: 2 + args: + tensor-parallel-size: 2 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + +override_2p5d_tp2_tp2_c120: + name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-agentic" + resources: + spread_workers: true + frontend: + num_additional_frontends: 4 + roles: + prefill: + nodes: 2 + workers: 2 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + decode: + nodes: 5 + workers: 5 + gpus: 2 + args: + tensor-parallel-size: 2 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + +override_2p5d_tp2_tp2_c120_eval: + name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-eval-agentic" + resources: + spread_workers: true + frontend: + num_additional_frontends: 4 + roles: + prefill: + nodes: 2 + workers: 2 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + decode: + nodes: 5 + workers: 5 + gpus: 2 + args: + tensor-parallel-size: 2 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 1e03899e43..c03a95f2c5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5605,8 +5605,8 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 4 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml" - - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tep4_tp4_c1" + - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tep4_tp4_c1_eval" decode: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -5618,8 +5618,8 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml" - - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p3d_dep4_tp4_c24" + - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p3d_dep4_tp4_c24_eval" decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -5631,8 +5631,8 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml" - - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tp2_tp4_c20_c24" + - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tp2_tp4_c20_c24_eval" decode: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -5644,8 +5644,8 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml" - - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p3d_tp2_tp2_c48" + - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p3d_tp2_tp2_c48_eval" decode: { num-worker: 3, tp: 2, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -5657,8 +5657,8 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml" - - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p5d_tp2_tp2_c120" + - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p5d_tp2_tp2_c120_eval" decode: { num-worker: 5, tp: 2, ep: 1, dp-attn: false } minimaxm3-fp4-b300-trtllm-agentic-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 @@ -5777,7 +5777,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml:override_tp4" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: vllm-simple } @@ -5789,7 +5789,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml:override_tp4_vllm_simple" - spec-decoding: mtp kv-offloading: none conc-list: [1] @@ -5800,7 +5800,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml:override_tp8" minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: image: vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6 @@ -5828,7 +5828,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: ep: 4 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml:override_1p1d_tp8_c1" decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -5840,7 +5840,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: ep: 4 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml:override_1p2d_c8_c16" decode: { num-worker: 2, tp: 4, ep: 4, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -5852,7 +5852,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: ep: 4 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml:override_1p1d_c24" decode: { num-worker: 1, tp: 4, ep: 4, dp-attn: false } minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: @@ -5880,7 +5880,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c5_b5" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } @@ -5892,7 +5892,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c10_b10" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } @@ -5904,7 +5904,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c15_b15" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } @@ -5916,7 +5916,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c20_b20" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } @@ -5928,7 +5928,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c25_b25" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } @@ -5940,7 +5940,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c30_b30" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } @@ -5952,7 +5952,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c40_b40" dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 From 39960deade91321fcc47bf45558f32c6a22aaff1 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:37:33 -0500 Subject: [PATCH 076/105] refactor(agentx): consolidate Qwen3.5 multi-node AgentX recipes into override variants Each per-configuration recipe becomes an `override_` block over a shared `base` in one `*-variants.yaml` per master-config entry, and the master entries select it with `CONFIG_FILE=...:override_`. Every selected variant resolves, through the pinned srtctl, to exactly the recipe it replaces, including its original `name`. Power recipes with top-level telemetry stay standalone because launchers detect them as text. --- ...2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml | 179 -------- ...2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml | 179 -------- ...2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml | 179 -------- ...2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml | 179 -------- ...2-colocated-c56-replayssm-mtp-hicache.yaml | 180 -------- ...p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml | 179 -------- ...p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml | 179 -------- ...1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml | 179 -------- ...-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml | 179 -------- .../b300-fp8/agentx/disagg-variants.yaml | 336 +++++++++++++++ .../agentx/agg-tp2ep2-mtp-hicache-cap48.yaml | 115 ----- .../agg-tp2ep2-mtp-hicache-k3-baseline.yaml | 116 ----- .../agentx/agg-tp2ep2-mtp-hicache.yaml | 117 ----- .../gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml | 111 ----- .../gb200-fp4/agentx/agg-tp4-mtp-parity.yaml | 114 ----- .../sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml | 111 ----- .../sglang/gb200-fp4/agentx/agg-variants.yaml | 249 +++++++++++ .../gb300-fp4/agentx/agg-pareto-variants.yaml | 186 ++++++++ .../agg-tp2-c1-mtp-hicache-jid2530006.yaml | 87 ---- .../agg-tp2-c24-mtp-hicache-jid2530012.yaml | 87 ---- .../agg-tp2-c32-mtp-hicache-jid2530013.yaml | 87 ---- .../agg-tp2-c40-mtp-hicache-jid2530015.yaml | 87 ---- ...-c44-b1-mtp-hicache-nightly-c20260831.yaml | 132 ------ ...-c44-b2-mtp-hicache-nightly-c20260831.yaml | 135 ------ .../agg-tp2-c48-mtp-hicache-jid2530017.yaml | 87 ---- .../agg-tp2-c52-mtp-hicache-jid2527406.yaml | 87 ---- .../agg-tp2-c64-mtp-hicache-jid2527410.yaml | 87 ---- ...8-c7-b1-mtp-hicache-nightly-c20260831.yaml | 132 ------ .../sglang/gb300-fp4/agentx/agg-variants.yaml | 113 +++++ ...p2-c72-mtp-hicache-session-jid2527415.yaml | 211 --------- ...4-c128-mtp-hicache-session-jid2527417.yaml | 206 --------- ...p4-c16-mtp-hicache-session-jid2530027.yaml | 206 --------- ...p4-c32-mtp-hicache-session-jid2530028.yaml | 206 --------- ...p4-c64-mtp-hicache-session-jid2530029.yaml | 206 --------- ...tp4-c8-mtp-hicache-session-jid2530030.yaml | 206 --------- ...p4-c96-mtp-hicache-session-jid2527409.yaml | 206 --------- ...p4-c704-mtp-hicache-nightly-c20260831.yaml | 250 ----------- ...p4-c565-mtp-hicache-nightly-c20260831.yaml | 250 ----------- .../agentx/disagg-pp-pareto-variants.yaml | 287 ++++++++++++ .../gb300-fp4/agentx/disagg-variants.yaml | 276 ++++++++++++ ...g-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml | 222 ---------- ...gg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml | 219 ---------- ...g-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml | 221 ---------- ...2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml | 236 ---------- ...3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml | 220 ---------- ...3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml | 223 ---------- .../gb300-fp4/agentx/disagg-variants.yaml | 408 ++++++++++++++++++ configs/nvidia-master.yaml | 80 ++-- 48 files changed, 1895 insertions(+), 6632 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml deleted file mode 100644 index 22cf32d1b7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated TP2+EP2 prefill and decode with write-through HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - enable-hierarchical-cache: true - hicache-size: 298 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through - decode: - nodes: colocate - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.88 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - cuda-graph-max-bs-decode: 32 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 32 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml deleted file mode 100644 index 494ae4b77c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated TP2+EP2 prefill and decode with write-through HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - enable-hierarchical-cache: true - hicache-size: 298 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through - decode: - nodes: colocate - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.88 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - cuda-graph-max-bs-decode: 48 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 40 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml deleted file mode 100644 index ba725e8464..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated TP2+EP2 prefill and decode with write-through HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - enable-hierarchical-cache: true - hicache-size: 298 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through - decode: - nodes: colocate - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.88 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - cuda-graph-max-bs-decode: 48 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 44 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml deleted file mode 100644 index 0b66b22cf3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated TP2+EP2 prefill and decode with write-through HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - enable-hierarchical-cache: true - hicache-size: 298 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through - decode: - nodes: colocate - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.92 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - cuda-graph-max-bs-decode: 48 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 48 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache.yaml deleted file mode 100644 index 2b48d3b7fc..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache.yaml +++ /dev/null @@ -1,180 +0,0 @@ -# Colocated TP2+EP2 prefill and decode with write-through HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - enable-hierarchical-cache: true - hicache-size: 298 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through - decode: - nodes: colocate - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.92 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - enable-linear-replayssm-spec: true - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - cuda-graph-max-bs-decode: 48 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 56 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml deleted file mode 100644 index 3b0e6a8083..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated prefill and decode with prefill HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - enable-hierarchical-cache: true - hicache-size: 72 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through_selective - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - cuda-graph-max-bs-decode: 16 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 12 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml deleted file mode 100644 index 69912c3f58..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated TP4 prefill and decode with prefill HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - enable-hierarchical-cache: true - hicache-size: 72 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through_selective - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - cuda-graph-max-bs-decode: 32 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 24 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml deleted file mode 100644 index 3273e19273..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated prefill and decode with prefill HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - enable-hierarchical-cache: true - hicache-size: 72 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through_selective - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - cuda-graph-max-bs-decode: 16 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 4 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml deleted file mode 100644 index 122687183c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated prefill and decode with prefill HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 4 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - enable-hierarchical-cache: true - hicache-size: 72 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through_selective - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - cuda-graph-max-bs-decode: 32 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 32 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..71394d6ae5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml @@ -0,0 +1,336 @@ +# AgentX qwen3.5 sglang b300-fp8 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 + identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e + slurm: + time_limit: '4:00:00' + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID + engine: sglang + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + disaggregation-decode-extra-slots: 0 + disaggregation-decode-retraction-backup: cpu_tensor + sbatch_directives: + mem: '0' + cpus-per-task: '144' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + ENABLE_AGENTX_POWER: '0' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# Colocated prefill and decode with prefill HiCache. +override_tp4_tp4_c4: + name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + max-running-requests: 32 + hicache-size: 72 + hicache-write-policy: write_through_selective + decode: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + max-running-requests: 32 + cuda-graph-max-bs-decode: 16 + benchmark: + concurrencies: + - 4 + +# Colocated prefill and decode with prefill HiCache. +override_tp4_tp4_c12: + name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + max-running-requests: 32 + hicache-size: 72 + hicache-write-policy: write_through_selective + decode: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + max-running-requests: 32 + cuda-graph-max-bs-decode: 16 + benchmark: + concurrencies: + - 12 + +# Colocated TP4 prefill and decode with prefill HiCache. +override_tp4_tp4_c24: + name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + max-running-requests: 32 + hicache-size: 72 + hicache-write-policy: write_through_selective + decode: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + benchmark: + concurrencies: + - 24 + +# Colocated prefill and decode with prefill HiCache. +override_tp4ep4_tp4_c32: + name: qwen35-b300-disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + max-running-requests: 32 + hicache-size: 72 + hicache-write-policy: write_through_selective + decode: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + benchmark: + concurrencies: + - 32 + +# Colocated TP2+EP2 prefill and decode with write-through HiCache. +override_tp2ep2_tp2ep2_c32: + name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache + roles: + decode: + args: + mem-fraction-static: 0.88 + cuda-graph-max-bs-decode: 32 + benchmark: + concurrencies: + - 32 + +# Colocated TP2+EP2 prefill and decode with write-through HiCache. +override_tp2ep2_tp2ep2_c40: + name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache + roles: + decode: + args: + mem-fraction-static: 0.88 + cuda-graph-max-bs-decode: 48 + benchmark: + concurrencies: + - 40 + +# Colocated TP2+EP2 prefill and decode with write-through HiCache. +override_tp2ep2_tp2ep2_c44: + name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache + roles: + decode: + args: + mem-fraction-static: 0.88 + cuda-graph-max-bs-decode: 48 + benchmark: + concurrencies: + - 44 + +# Colocated TP2+EP2 prefill and decode with write-through HiCache. +override_tp2ep2_tp2ep2_c48: + name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache + roles: + decode: + args: + mem-fraction-static: 0.92 + cuda-graph-max-bs-decode: 48 + benchmark: + concurrencies: + - 48 + +# Colocated TP2+EP2 prefill and decode with write-through HiCache. +override_tp2ep2_tp2ep2_c56_replayssm: + name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache + roles: + decode: + args: + mem-fraction-static: 0.92 + enable-linear-replayssm-spec: true + cuda-graph-max-bs-decode: 48 + benchmark: + concurrencies: + - 56 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml deleted file mode 100644 index fe54b630a5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml +++ /dev/null @@ -1,115 +0,0 @@ -schema: 2 -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-cap48 - -model: {path: qwen3.5-fp4, container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b, precision: fp4} -identity: - model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} - container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} -resources: {gpu_type: gb200, gpus_per_node: 4} -services: - - name: nats - type: nats - options: - max_payload_mb: 8 -dynamo: - install: true - source: - wheel: 1.4.2 - request_plane: tcp -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-radix-cache-strategy: extra_buffer_lazy - mamba-track-interval: 1048576 - attention-backend: trtllm_mha - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 8 - cuda-graph-max-bs: 48 - max-running-requests: 48 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.88 - max-mamba-cache-size: 192 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 2.5 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - hicache-write-policy: write_through_selective - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - watchdog-timeout: 1000000 - enable-metrics: true - enable-cache-report: true - -sbatch_directives: {mem: "0", cpus-per-task: "144"} -srun_options: {mem: "0", container-remap-root: ""} -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "2" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml deleted file mode 100644 index 919890bc13..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml +++ /dev/null @@ -1,116 +0,0 @@ -schema: 2 -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-k3-baseline - -model: {path: qwen3.5-fp4, container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b, precision: fp4} -identity: - model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} - container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} -resources: {gpu_type: gb200, gpus_per_node: 4} -services: - - name: nats - type: nats - options: - max_payload_mb: 8 -dynamo: - install: true - source: - wheel: 1.4.2 - request_plane: tcp -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-radix-cache-strategy: extra_buffer_lazy - mamba-track-interval: 1048576 - attention-backend: trtllm_mha - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 8 - cuda-graph-max-bs: 64 - max-running-requests: 80 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.88 - max-mamba-cache-size: 320 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 1 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 2.5 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - hicache-write-policy: write_through_selective - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - watchdog-timeout: 1000000 - enable-metrics: true - enable-cache-report: true - -sbatch_directives: {mem: "0", cpus-per-task: "144"} -srun_options: {mem: "0", container-remap-root: ""} -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "2" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml deleted file mode 100644 index baa7b9e34f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml +++ /dev/null @@ -1,117 +0,0 @@ -schema: 2 -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-k5 - -model: {path: qwen3.5-fp4, container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b, precision: fp4} -identity: - model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} - container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} -resources: {gpu_type: gb200, gpus_per_node: 4} -services: - - name: nats - type: nats - options: - max_payload_mb: 8 -dynamo: - install: true - source: - wheel: 1.4.2 - request_plane: tcp -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-radix-cache-strategy: extra_buffer_lazy - mamba-track-interval: 1048576 - attention-backend: trtllm_mha - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 8 - cuda-graph-max-bs: 64 - max-running-requests: 80 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.88 - # extra_buffer_lazy uses four physical Mamba state slots per running request. - # This layout is the measured K5 middle frontier through C28. - max-mamba-cache-size: 320 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 2.5 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - hicache-write-policy: write_through_selective - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - watchdog-timeout: 1000000 - enable-metrics: true - enable-cache-report: true - -sbatch_directives: {mem: "0", cpus-per-task: "144"} -srun_options: {mem: "0", container-remap-root: ""} -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "2" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml deleted file mode 100644 index 81f1bae1b7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml +++ /dev/null @@ -1,111 +0,0 @@ -schema: 2 -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5-no-symm - -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b - precision: fp4 - -identity: - model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} - container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} - -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: gb200 - gpus_per_node: 4 -services: - - name: nats - type: nats - options: - max_payload_mb: 8 -dynamo: - install: true - source: - wheel: 1.4.2 - request_plane: tcp -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - - env: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: false - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - cuda-graph-max-bs: 64 - max-running-requests: 80 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.80 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - enable-metrics: true - enable-cache-report: true - -sbatch_directives: {mem: "0", cpus-per-task: "144"} -srun_options: {mem: "0", container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "4" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml deleted file mode 100644 index 35abe029cc..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml +++ /dev/null @@ -1,114 +0,0 @@ -schema: 2 -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5-parity - -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b - precision: fp4 - -identity: - model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} - container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} - -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: gb200 - gpus_per_node: 4 -services: - - name: nats - type: nats - options: - max_payload_mb: 8 -dynamo: - install: true - source: - wheel: 1.4.2 - request_plane: tcp -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - - env: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer_lazy - mamba-track-interval: 1048576 - attention-backend: trtllm_mha - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 8 - cuda-graph-max-bs: 64 - max-running-requests: 80 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.80 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - enable-metrics: true - enable-cache-report: true - -sbatch_directives: {mem: "0", cpus-per-task: "144"} -srun_options: {mem: "0", container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "4" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml deleted file mode 100644 index bf14c75b64..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml +++ /dev/null @@ -1,111 +0,0 @@ -schema: 2 -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5 - -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b - precision: fp4 - -identity: - model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} - container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} - -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: gb200 - gpus_per_node: 4 -services: - - name: nats - type: nats - options: - max_payload_mb: 8 -dynamo: - install: true - source: - wheel: 1.4.2 - request_plane: tcp -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - - env: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - cuda-graph-max-bs: 64 - max-running-requests: 80 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.80 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - enable-metrics: true - enable-cache-report: true - -sbatch_directives: {mem: "0", cpus-per-task: "144"} -srun_options: {mem: "0", container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "4" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..c2e9769d2b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml @@ -0,0 +1,249 @@ +# AgentX qwen3.5 sglang gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b + precision: fp4 + identity: + model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} + container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} + slurm: {time_limit: "8:00:00"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: gb200 + gpus_per_node: 4 + services: + - name: nats + type: nats + options: + max_payload_mb: 8 + dynamo: + install: true + source: + wheel: 1.4.2 + request_plane: tcp + environment: + PIP_BREAK_SYSTEM_PACKAGES: "1" + frontend: + type: dynamo + enable_multiple_frontends: false + env: {} + args: + router-mode: round-robin + router-session-affinity-ttl-secs: 3600 + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + env: + PYTHONNOUSERSITE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_ENABLE_SPEC_V2: "1" + SGL_ENABLE_JIT_DEEPGEMM: "false" + TORCH_CUDA_ARCH_LIST: "10.0" + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model/ + trust-remote-code: true + data-parallel-size: 1 + enable-symm-mem: false + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 1048576 + attention-backend: trtllm_mha + linear-attn-decode-backend: flashinfer + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + cuda-graph-max-bs: 64 + max-running-requests: 80 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + mamba-max-states-per-path: 1 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + weight-loader-drop-cache-after-load: false + model-loader-extra-config: '{"enable_multithread_load":true}' + enable-metrics: true + enable-cache-report: true + sbatch_directives: {mem: "0", cpus-per-task: "144"} + srun_options: {mem: "0", container-remap-root: ""} + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' + +override_tp4: + name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5 + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + enable-symm-mem: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + mem-fraction-static: 0.80 + max-mamba-cache-size: 360 + benchmark: + env: + TP: "4" + +override_tp4_parity: + name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5-parity + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + enable-symm-mem: true + mamba-scheduler-strategy: extra_buffer_lazy + linear-attn-prefill-backend: flashinfer + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 8 + mem-fraction-static: 0.80 + max-mamba-cache-size: 360 + benchmark: + env: + TP: "4" + +override_tp4_no_symm: + name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5-no-symm + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + mem-fraction-static: 0.80 + max-mamba-cache-size: 360 + benchmark: + env: + TP: "4" + +override_tp2ep2_hicache: + name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-k5 + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + expert-parallel-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-radix-cache-strategy: extra_buffer_lazy + linear-attn-prefill-backend: flashinfer + speculative-moe-runner-backend: flashinfer_trtllm + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 8 + mem-fraction-static: 0.88 + # extra_buffer_lazy uses four physical Mamba state slots per running request. + # This layout is the measured K5 middle frontier through C28. + max-mamba-cache-size: 320 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 2.5 + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-write-policy: write_through_selective + watchdog-timeout: 1000000 + benchmark: + env: + TP: "2" + +override_tp2ep2_hicache_cap48: + name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-cap48 + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + expert-parallel-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-radix-cache-strategy: extra_buffer_lazy + linear-attn-prefill-backend: flashinfer + speculative-moe-runner-backend: flashinfer_trtllm + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 8 + cuda-graph-max-bs: 48 + max-running-requests: 48 + mem-fraction-static: 0.88 + max-mamba-cache-size: 192 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 2.5 + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-write-policy: write_through_selective + watchdog-timeout: 1000000 + benchmark: + env: + TP: "2" + +override_tp2ep2_hicache_k3_baseline: + name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-k3-baseline + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + expert-parallel-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-radix-cache-strategy: extra_buffer_lazy + linear-attn-prefill-backend: flashinfer + speculative-moe-runner-backend: flashinfer_trtllm + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 8 + mem-fraction-static: 0.88 + max-mamba-cache-size: 320 + tokenizer-worker-num: 1 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 2.5 + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-write-policy: write_through_selective + watchdog-timeout: 1000000 + benchmark: + env: + TP: "2" diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml new file mode 100644 index 0000000000..c65a482875 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml @@ -0,0 +1,186 @@ +# AgentX qwen3.5 sglang gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml:override_. + +schema: 2 + +base: + model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + precision: fp4 + slurm: + time_limit: '8:00:00' + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 2 + frontend: + type: sglang-router + enable_multiple_frontends: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-dense-tp-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + moe-a2a-backend: none + load-balance-method: round_robin + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 1536 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 6 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + speculative-moe-runner-backend: flashinfer_trtllm + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + mem-fraction-static: 0.85 + max-running-requests: 1 + pp-max-micro-batch-size: 1 + prefill-max-requests: 1 + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-size: 32 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true + cuda-graph-max-bs-decode: 1 + cuda-graph-bs-decode: + - 1 + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +override_tp2_c44_b1: + name: agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831 + roles: + agg: + env: + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/flashinfer-cache + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/sglang-cache + args: + prefill-decode-interval: 0 + +override_tp2_c44_b2: + name: agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831 + roles: + agg: + env: + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/flashinfer-cache + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/sglang-cache + args: + prefill-decode-interval: 0 + max-running-requests: 2 + pp-max-micro-batch-size: 2 + prefill-max-requests: 2 + cuda-graph-max-bs-decode: 2 + cuda-graph-bs-decode: + - 1 + - 2 + hicache-size: 128 + +override_tp8_c7_b1: + name: agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831 + resources: + gpus_per_node: 4 + roles: + agg: + nodes: 2 + gpus: 8 + env: + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/c7-mrr1/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/c7-mrr1/flashinfer-cache + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/c7-mrr1/sglang-cache + args: + tensor-parallel-size: 8 + moe-dense-tp-size: 8 + moe-runner-backend: flashinfer_cutedsl + speculative-moe-runner-backend: flashinfer_cutedsl + disable-prefill-cuda-graph: true + benchmark: + env: + IS_MULTINODE: 'true' + TP: '8' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml deleted file mode 100644 index 4d71db73f4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c1-mtp-hicache-jid2530006 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml deleted file mode 100644 index cb64f23657..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c24-mtp-hicache-jid2530012 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml deleted file mode 100644 index 2271ef3e29..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c32-mtp-hicache-jid2530013 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml deleted file mode 100644 index 3335faf1b3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c40-mtp-hicache-jid2530015 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml deleted file mode 100644 index 1ab4c4a73d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml +++ /dev/null @@ -1,132 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831 -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router - enable_multiple_frontends: false -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/flashinfer-cache - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/sglang-cache - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 1 - moe-dense-tp-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - moe-a2a-backend: none - load-balance-method: round_robin - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 8192 - mamba-max-states-per-path: 3 - mamba-ssm-dtype: bfloat16 - max-mamba-cache-size: 1536 - context-length: 262144 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: triton - disable-shared-experts-fusion: true - speculative-algorithm: NEXTN - speculative-num-steps: 6 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - speculative-moe-runner-backend: flashinfer_trtllm - speculative-moe-a2a-backend: none - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - prefill-decode-interval: 0 - mem-fraction-static: 0.85 - max-running-requests: 1 - pp-max-micro-batch-size: 1 - prefill-max-requests: 1 - stream-interval: 20 - decode-log-interval: 10 - watchdog-timeout: 1000000 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-size: 32 - enable-metrics: true - enable-linear-replayssm-spec: true - disable-attn-tp-gather: true - cuda-graph-max-bs-decode: 1 - cuda-graph-bs-decode: - - 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml deleted file mode 100644 index b3dd048818..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml +++ /dev/null @@ -1,135 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831 -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router - enable_multiple_frontends: false -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: - /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: - /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/flashinfer-cache - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/sglang-cache - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 1 - moe-dense-tp-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - moe-a2a-backend: none - load-balance-method: round_robin - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 8192 - mamba-max-states-per-path: 3 - mamba-ssm-dtype: bfloat16 - max-mamba-cache-size: 1536 - context-length: 262144 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: triton - disable-shared-experts-fusion: true - speculative-algorithm: NEXTN - speculative-num-steps: 6 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - speculative-moe-runner-backend: flashinfer_trtllm - speculative-moe-a2a-backend: none - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - prefill-decode-interval: 0 - mem-fraction-static: 0.85 - max-running-requests: 2 - pp-max-micro-batch-size: 2 - prefill-max-requests: 2 - cuda-graph-max-bs-decode: 2 - cuda-graph-bs-decode: - - 1 - - 2 - stream-interval: 20 - decode-log-interval: 10 - watchdog-timeout: 1000000 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-size: 128 - enable-metrics: true - enable-linear-replayssm-spec: true - disable-attn-tp-gather: true -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml deleted file mode 100644 index b3d34533f4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c48-mtp-hicache-jid2530017 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml deleted file mode 100644 index cbd83c1850..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c52-mtp-hicache-jid2527406 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml deleted file mode 100644 index 4a6f68d048..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c64-mtp-hicache-jid2527410 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml deleted file mode 100644 index cb75cb22f3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml +++ /dev/null @@ -1,132 +0,0 @@ -schema: 2 -name: agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831 -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -frontend: - type: sglang-router - enable_multiple_frontends: false -engine: sglang -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - env: - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/c7-mrr1/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/c7-mrr1/flashinfer-cache - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/c7-mrr1/sglang-cache - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 1 - moe-dense-tp-size: 8 - enable-dp-attention: false - enable-dp-lm-head: false - moe-a2a-backend: none - load-balance-method: round_robin - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 8192 - mamba-max-states-per-path: 3 - mamba-ssm-dtype: bfloat16 - max-mamba-cache-size: 1536 - context-length: 262144 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - linear-attn-decode-backend: triton - disable-shared-experts-fusion: true - speculative-algorithm: NEXTN - speculative-num-steps: 6 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - speculative-moe-runner-backend: flashinfer_cutedsl - speculative-moe-a2a-backend: none - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - mem-fraction-static: 0.85 - max-running-requests: 1 - pp-max-micro-batch-size: 1 - prefill-max-requests: 1 - cuda-graph-max-bs-decode: 1 - cuda-graph-bs-decode: - - 1 - disable-prefill-cuda-graph: true - stream-interval: 20 - decode-log-interval: 10 - watchdog-timeout: 1000000 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-size: 32 - enable-metrics: true - enable-linear-replayssm-spec: true - disable-attn-tp-gather: true -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - TP: '8' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..946105d17c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -0,0 +1,113 @@ +# AgentX qwen3.5 sglang gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 + slurm: + time_limit: '8:00:00' + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 2 + frontend: + type: sglang-router + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +override_c1: + name: agg-gb300-tp2-c1-mtp-hicache-jid2530006 + +override_c24: + name: agg-gb300-tp2-c24-mtp-hicache-jid2530012 + +override_c32: + name: agg-gb300-tp2-c32-mtp-hicache-jid2530013 + +override_c40: + name: agg-gb300-tp2-c40-mtp-hicache-jid2530015 + +override_c48: + name: agg-gb300-tp2-c48-mtp-hicache-jid2530017 + +override_c52: + name: agg-gb300-tp2-c52-mtp-hicache-jid2527406 + +override_c64: + name: agg-gb300-tp2-c64-mtp-hicache-jid2527410 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml deleted file mode 100644 index e5d160a211..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml +++ /dev/null @@ -1,211 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - max-mamba-cache-size: 320 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - mamba-max-states-per-path: 1 - decode: - nodes: 1 - workers: 1 - gpus: 2 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 80 - max-running-requests: 80 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - decode-log-interval: 10 - mamba-max-states-per-path: -1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml deleted file mode 100644 index 3c6ffd223b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml +++ /dev/null @@ -1,206 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - decode: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 160 - watchdog-timeout: 1000000 - decode-log-interval: 10 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml deleted file mode 100644 index cfc162993a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml +++ /dev/null @@ -1,206 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - decode: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 64 - watchdog-timeout: 1000000 - decode-log-interval: 10 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml deleted file mode 100644 index 58a83e4df2..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml +++ /dev/null @@ -1,206 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - decode: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 64 - watchdog-timeout: 1000000 - decode-log-interval: 10 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml deleted file mode 100644 index 6645e6c1ee..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml +++ /dev/null @@ -1,206 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - decode: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 80 - watchdog-timeout: 1000000 - decode-log-interval: 10 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml deleted file mode 100644 index 67d10772d9..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml +++ /dev/null @@ -1,206 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - decode: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 64 - watchdog-timeout: 1000000 - decode-log-interval: 10 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml deleted file mode 100644 index b5973a79e0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml +++ /dev/null @@ -1,206 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - decode: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - decode-log-interval: 10 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml deleted file mode 100644 index 96fc807eaa..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml +++ /dev/null @@ -1,250 +0,0 @@ -schema: 2 -name: disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831 -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 -dynamo: - install: true - source: - rev: 61c37bc33c3e88da3facaa39fa2b5caf616740ba -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 1 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-replica-sync: true - router-queue-threshold: None - router-temperature: '1.0' -engine: sglang -roles: - prefill: - nodes: 3 - workers: 3 - gpus: 4 - env: - # The 0831 image contains the PP+spec runtime merged by SGLang #35758, - # but still carries the pre-merge PP+spec assert in validation_hook.py. - PYTHONOPTIMIZE: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-upstream-main/prefill-deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-upstream-main/prefill-flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '8192' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_PP_LAYER_PARTITION: 16,16,16,12 - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - SGLANG_FLASHINFER_AUTOTUNE_EXTEND: '1' - SGLANG_CACHE_DIR: /tmp/agentx-upstream-main/prefill-sglang-cache - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 1 - pipeline-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false - disable-overlap-schedule: true - enable-dynamic-chunking: false - mamba-ssm-dtype: bfloat16 - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 1048576 - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1536 - max-running-requests: 128 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - linear-attn-decode-backend: triton - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - watchdog-timeout: 1000000 - log-level: info - nccl-port: 29500 - scheduler-recv-interval: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - moe-dense-tp-size: 1 - disable-shared-experts-fusion: true - moe-a2a-backend: none - speculative-moe-a2a-backend: none - speculative-moe-runner-backend: flashinfer_trtllm - pp-async-batch-depth: 1 - disable-cuda-graph: true - decode: - nodes: 2 - workers: 2 - gpus: 4 - env: - SGLANG_USE_SYMM_MEM_DP_SYNC: 'true' - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '4' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-upstream-main/decode-deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-upstream-main/decode-flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_CACHE_DIR: /tmp/agentx-upstream-main/decode-sglang-cache - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - data-parallel-size: 4 - expert-parallel-size: 4 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: round_robin - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - linear-attn-decode-backend: triton - disable-shared-experts-fusion: true - moe-a2a-backend: flashinfer - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - speculative-draft-model-quantization: nvfp4_online - speculative-moe-runner-backend: flashinfer_trtllm_routed - speculative-moe-a2a-backend: flashinfer - chunked-prefill-size: 4096 - mem-fraction-static: 0.7 - max-mamba-cache-size: 468 - max-running-requests: 320 - cuda-graph-max-bs: 80 - disaggregation-decode-extra-slots: 2 - stream-interval: 30 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 32 - decode-log-interval: 30 - watchdog-timeout: 1000000 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml deleted file mode 100644 index 9f8f0e1b76..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml +++ /dev/null @@ -1,250 +0,0 @@ -schema: 2 -name: disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831 -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 -dynamo: - install: true - source: - rev: 61c37bc33c3e88da3facaa39fa2b5caf616740ba -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 1 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-replica-sync: true - router-queue-threshold: None - router-temperature: '1.0' -engine: sglang -roles: - prefill: - nodes: 3 - workers: 3 - gpus: 4 - env: - # The 0831 image contains the PP+spec runtime merged by SGLang #35758, - # but still carries the pre-merge PP+spec assert in validation_hook.py. - PYTHONOPTIMIZE: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '8192' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_PP_LAYER_PARTITION: 16,16,16,12 - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - SGLANG_FLASHINFER_AUTOTUNE_EXTEND: '1' - SGLANG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-sglang-cache - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 1 - pipeline-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false - disable-overlap-schedule: true - enable-dynamic-chunking: false - mamba-ssm-dtype: bfloat16 - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 1048576 - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1536 - max-running-requests: 128 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - speculative-algorithm: NEXTN - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - linear-attn-decode-backend: triton - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - watchdog-timeout: 1000000 - log-level: info - nccl-port: 29500 - scheduler-recv-interval: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - moe-dense-tp-size: 1 - disable-shared-experts-fusion: true - moe-a2a-backend: none - speculative-moe-a2a-backend: none - speculative-moe-runner-backend: flashinfer_trtllm - pp-async-batch-depth: 1 - disable-cuda-graph: true - decode: - nodes: 4 - workers: 4 - gpus: 4 - env: - SGLANG_USE_SYMM_MEM_DP_SYNC: 'true' - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '4' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/decode-deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-followup/c565-cutedsl-g3072/decode-flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/decode-sglang-cache - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - data-parallel-size: 4 - expert-parallel-size: 4 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: round_robin - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - linear-attn-decode-backend: triton - disable-shared-experts-fusion: true - moe-a2a-backend: flashinfer - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - speculative-draft-model-quantization: nvfp4_online - speculative-moe-runner-backend: flashinfer_trtllm_routed - speculative-moe-a2a-backend: flashinfer - chunked-prefill-size: 4096 - mem-fraction-static: 0.7 - max-mamba-cache-size: 468 - max-running-requests: 320 - cuda-graph-max-bs: 80 - disaggregation-decode-extra-slots: 2 - stream-interval: 30 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 32 - decode-log-interval: 30 - watchdog-timeout: 1000000 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml new file mode 100644 index 0000000000..3caef43cbc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml @@ -0,0 +1,287 @@ +# AgentX qwen3.5 sglang gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml:override_. + +schema: 2 + +base: + model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + precision: fp4 + dynamo: + install: true + source: + rev: 61c37bc33c3e88da3facaa39fa2b5caf616740ba + slurm: + time_limit: '8:00:00' + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 8 + frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-replica-sync: true + router-queue-threshold: None + router-temperature: '1.0' + engine: sglang + roles: + prefill: + nodes: 3 + workers: 3 + gpus: 4 + env: + # The 0831 image contains the PP+spec runtime merged by SGLang #35758, + # but still carries the pre-merge PP+spec assert in validation_hook.py. + PYTHONOPTIMIZE: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '8192' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_PP_LAYER_PARTITION: 16,16,16,12 + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_FLASHINFER_AUTOTUNE_EXTEND: '1' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 1 + pipeline-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + enable-symm-mem: false + disable-overlap-schedule: true + enable-dynamic-chunking: false + mamba-ssm-dtype: bfloat16 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 1048576 + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1536 + max-running-requests: 128 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + linear-attn-decode-backend: triton + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + watchdog-timeout: 1000000 + log-level: info + nccl-port: 29500 + scheduler-recv-interval: 1 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + moe-dense-tp-size: 1 + disable-shared-experts-fusion: true + moe-a2a-backend: none + speculative-moe-a2a-backend: none + speculative-moe-runner-backend: flashinfer_trtllm + pp-async-batch-depth: 1 + disable-cuda-graph: true + decode: + gpus: 4 + env: + SGLANG_USE_SYMM_MEM_DP_SYNC: 'true' + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '4' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + data-parallel-size: 4 + expert-parallel-size: 4 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: round_robin + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + moe-a2a-backend: flashinfer + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-draft-model-quantization: nvfp4_online + speculative-moe-runner-backend: flashinfer_trtllm_routed + speculative-moe-a2a-backend: flashinfer + chunked-prefill-size: 4096 + mem-fraction-static: 0.7 + max-mamba-cache-size: 468 + max-running-requests: 320 + cuda-graph-max-bs: 80 + disaggregation-decode-extra-slots: 2 + stream-interval: 30 + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 32 + decode-log-interval: 30 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +override_3p4d_c565: + name: disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831 + roles: + prefill: + env: + SGLANG_DG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-flashinfer-cache + SGLANG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-sglang-cache + args: + speculative-num-steps: 4 + speculative-num-draft-tokens: 5 + decode: + nodes: 4 + workers: 4 + env: + SGLANG_DG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/decode-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-followup/c565-cutedsl-g3072/decode-flashinfer-cache + SGLANG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/decode-sglang-cache + args: + speculative-num-steps: 4 + speculative-num-draft-tokens: 5 + +override_3p2d_c704: + name: disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831 + roles: + prefill: + env: + SGLANG_DG_CACHE_DIR: /tmp/agentx-upstream-main/prefill-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-upstream-main/prefill-flashinfer-cache + SGLANG_CACHE_DIR: /tmp/agentx-upstream-main/prefill-sglang-cache + args: + speculative-num-steps: 5 + speculative-num-draft-tokens: 6 + decode: + nodes: 2 + workers: 2 + env: + SGLANG_DG_CACHE_DIR: /tmp/agentx-upstream-main/decode-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-upstream-main/decode-flashinfer-cache + SGLANG_CACHE_DIR: /tmp/agentx-upstream-main/decode-sglang-cache + args: + speculative-num-steps: 5 + speculative-num-draft-tokens: 6 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..b38e93698f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,276 @@ +# AgentX qwen3.5 sglang gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 + dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + slurm: + time_limit: '8:00:00' + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 8 + frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID + engine: sglang + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + watchdog-timeout: 1000000 + decode-log-interval: 10 + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +override_tp4_tp4_c8: + name: disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030 + roles: + decode: + args: + cuda-graph-max-bs: 64 + +override_tp4_tp4_c16: + name: disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027 + roles: + decode: + args: + cuda-graph-max-bs: 64 + +override_tp4_tp4_c32: + name: disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028 + roles: + decode: + args: + cuda-graph-max-bs: 64 + +override_tp4_tp4_c64: + name: disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029 + roles: + decode: + args: + cuda-graph-max-bs: 80 + +override_tp4_tp4_c96: + name: disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409 + roles: + decode: + args: + cuda-graph-max-bs: 128 + +override_tp4_tp4_c128: + name: disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417 + roles: + decode: + args: + cuda-graph-max-bs: 160 + +override_tp2_tp2_c72: + name: disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415 + resources: + gpus_per_node: 2 + roles: + prefill: + gpus: 2 + args: + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + tensor-parallel-size: 2 + max-mamba-cache-size: 320 + mamba-max-states-per-path: 1 + decode: + gpus: 2 + args: + tensor-parallel-size: 2 + max-mamba-cache-size: 80 + max-running-requests: 80 + cuda-graph-max-bs: 128 + mamba-max-states-per-path: -1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml deleted file mode 100644 index 5f2df55f53..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml +++ /dev/null @@ -1,222 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 1 - env: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 8192 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 1 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - nodes: 1 - workers: 1 - gpus: 2 - env: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - enable_padding: true - enable_attention_dp: false - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.85 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 8 - max_num_tokens: 56 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 2 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 6 - stream_interval: 20 - tensor_parallel_size: 2 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml deleted file mode 100644 index ae71196e1e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml +++ /dev/null @@ -1,219 +0,0 @@ -schema: 2 -name: disagg-gb300-1p7d-dep4-tep8-c7-b1-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 8192 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 4 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - nodes: 14 - workers: 7 - gpus: 8 - env: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - enable_attention_dp: false - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.85 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 1 - max_num_tokens: 8 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 7 - stream_interval: 20 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml deleted file mode 100644 index 6c0b923ce1..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml +++ /dev/null @@ -1,221 +0,0 @@ -schema: 2 -name: disagg-gb300-2p2d-dep1-tep2-c52-b4-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 1 - workers: 2 - gpus: 1 - env: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 8192 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 1 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - nodes: 1 - workers: 2 - gpus: 2 - env: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - enable_attention_dp: false - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.85 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 4 - max_num_tokens: 28 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 2 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 6 - stream_interval: 20 - tensor_parallel_size: 2 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml deleted file mode 100644 index 5db9ff2de7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml +++ /dev/null @@ -1,236 +0,0 @@ -schema: 2 -name: disagg-gb300-2p3d-tep2-tep8-c96-b128-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 1 - workers: 2 - gpus: 2 - env: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: false - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 16384 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 2 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 2 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - - 16384 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - nodes: 6 - workers: 3 - gpus: 8 - env: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - enable_padding: true - enable_attention_dp: false - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.85 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 128 - max_num_tokens: 896 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 6 - stream_interval: 20 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml deleted file mode 100644 index 559ab778a5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml +++ /dev/null @@ -1,220 +0,0 @@ -schema: 2 -name: disagg-gb300-3p1d-dep4-dep16-c565-b8-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 3 - workers: 3 - gpus: 4 - env: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 8192 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 4 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - nodes: 4 - workers: 1 - gpus: 16 - env: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - enable_padding: true - enable_attention_dp: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.85 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 8 - max_num_tokens: 56 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 6 - stream_interval: 20 - tensor_parallel_size: 16 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml deleted file mode 100644 index e9e81c54fc..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml +++ /dev/null @@ -1,223 +0,0 @@ -schema: 2 -name: disagg-gb300-3p2d-dep4-dep4-c704-b32-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 3 - workers: 3 - gpus: 4 - env: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 8192 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 4 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - nodes: 2 - workers: 2 - gpus: 4 - env: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - enable_padding: true - enable_attention_dp: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 32 - max_num_tokens: 224 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 6 - stream_interval: 20 - tensor_parallel_size: 4 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..d607b6abae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,408 @@ +# AgentX qwen3.5 trtllm gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + slurm: + time_limit: 04:00:00 + health_check: + max_attempts: 540 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + engine: + type: trtllm + publish_events_and_metrics: false + roles: + prefill: + nodes: 1 + env: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: true + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + env: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: true + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + trust_remote_code: true + frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 + +override_1p7d_dep4_tep8_c7_b1: + name: disagg-gb300-1p7d-dep4-tep8-c7-b1-mtp-kvoffload + roles: + prefill: + workers: 1 + gpus: 4 + args: + moe_expert_parallel_size: 4 + tensor_parallel_size: 4 + decode: + nodes: 14 + workers: 7 + gpus: 8 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + max_batch_size: 1 + max_num_tokens: 8 + moe_expert_parallel_size: 8 + speculative_config: + max_draft_len: 7 + tensor_parallel_size: 8 + +override_1p1d_dep1_tep2_c44_b8: + name: disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload + roles: + prefill: + workers: 1 + gpus: 1 + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + args: + moe_expert_parallel_size: 1 + tensor_parallel_size: 1 + decode: + nodes: 1 + workers: 1 + gpus: 2 + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + max_batch_size: 8 + max_num_tokens: 56 + moe_expert_parallel_size: 2 + tensor_parallel_size: 2 + +override_2p2d_dep1_tep2_c52_b4: + name: disagg-gb300-2p2d-dep1-tep2-c52-b4-mtp-kvoffload + roles: + prefill: + workers: 2 + gpus: 1 + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + args: + moe_expert_parallel_size: 1 + tensor_parallel_size: 1 + decode: + nodes: 1 + workers: 2 + gpus: 2 + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + max_batch_size: 4 + max_num_tokens: 28 + moe_expert_parallel_size: 2 + tensor_parallel_size: 2 + +override_2p3d_tep2_tep8_c96_b128: + name: disagg-gb300-2p3d-tep2-tep8-c96-b128-mtp-kvoffload + roles: + prefill: + workers: 2 + gpus: 2 + args: + enable_attention_dp: false + max_num_tokens: 16384 + moe_expert_parallel_size: 2 + tensor_parallel_size: 2 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + - 16384 + decode: + nodes: 6 + workers: 3 + gpus: 8 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + max_batch_size: 128 + max_num_tokens: 896 + moe_expert_parallel_size: 8 + tensor_parallel_size: 8 + +override_3p1d_dep4_dep16_c565_b8: + name: disagg-gb300-3p1d-dep4-dep16-c565-b8-mtp-kvoffload + roles: + prefill: + nodes: 3 + workers: 3 + gpus: 4 + args: + moe_expert_parallel_size: 4 + tensor_parallel_size: 4 + decode: + nodes: 4 + workers: 1 + gpus: 16 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + max_batch_size: 8 + max_num_tokens: 56 + moe_expert_parallel_size: 16 + tensor_parallel_size: 16 + +override_3p2d_dep4_dep4_c704_b32: + name: disagg-gb300-3p2d-dep4-dep4-c704-b32-mtp-kvoffload + roles: + prefill: + nodes: 3 + workers: 3 + gpus: 4 + args: + moe_expert_parallel_size: 4 + tensor_parallel_size: 4 + decode: + nodes: 2 + workers: 2 + gpus: 4 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + free_gpu_memory_fraction: 0.8 + max_batch_size: 32 + max_num_tokens: 224 + moe_expert_parallel_size: 4 + tensor_parallel_size: 4 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c03a95f2c5..839f48a7b6 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4613,7 +4613,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_dep1_tep2_c44_b8" decode: num-worker: 1 tp: 2 @@ -4630,7 +4630,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p7d_dep4_tep8_c7_b1" decode: num-worker: 7 tp: 8 @@ -4647,7 +4647,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p2d_dep1_tep2_c52_b4" decode: num-worker: 2 tp: 2 @@ -4664,7 +4664,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 2 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p3d_tep2_tep8_c96_b128" decode: num-worker: 3 tp: 8 @@ -4681,7 +4681,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_3p1d_dep4_dep16_c565_b8" decode: num-worker: 1 tp: 16 @@ -4698,7 +4698,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_3p2d_dep4_dep4_c704_b32" decode: num-worker: 2 tp: 4 @@ -4729,7 +4729,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c1" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -4741,7 +4741,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c24" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -4753,7 +4753,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c32" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -4765,7 +4765,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c40" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -4777,7 +4777,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c48" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -4789,7 +4789,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c52" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -4801,7 +4801,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c64" # The seven disaggregated frontier points use six TP4 shapes plus one TP2 # shape. Stable X-Dynamo-Session-ID affinity replaces the removed conv-aware # routing message path. @@ -4830,7 +4830,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c8" decode: num-worker: 1 tp: 4 @@ -4846,7 +4846,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c16" decode: num-worker: 1 tp: 4 @@ -4862,7 +4862,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c32" decode: num-worker: 1 tp: 4 @@ -4878,7 +4878,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c64" decode: num-worker: 1 tp: 4 @@ -4894,7 +4894,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c96" decode: num-worker: 1 tp: 4 @@ -4910,7 +4910,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c128" decode: num-worker: 1 tp: 4 @@ -4926,7 +4926,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp2_tp2_c72" decode: num-worker: 1 tp: 2 @@ -4963,7 +4963,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml:override_3p2d_c704" decode: num-worker: 2 tp: 4 @@ -4981,7 +4981,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml:override_3p4d_c565" decode: num-worker: 4 tp: 4 @@ -5014,7 +5014,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml:override_tp2_c44_b2" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -5027,7 +5027,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml:override_tp2_c44_b1" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -5040,7 +5040,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml:override_tp8_c7_b1" qwen3.5-fp8-b300-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 @@ -5498,7 +5498,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp4" - spec-decoding: mtp kv-offloading: none conc-list: [2] @@ -5509,7 +5509,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp4_no_symm" - spec-decoding: mtp kv-offloading: none conc-list: [24] @@ -5520,7 +5520,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp4_parity" # Keep only the reproducible TP2/EP2 HiCache K5 transition point; C28 # was dominated by the published C28 result in the official PR sweep. - spec-decoding: mtp @@ -5534,7 +5534,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 2 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp2ep2_hicache" # Keep the published C32 interactivity anchor. The tighter capacity # layout becomes Pareto-relevant at C40 and remains deployable at C48. - spec-decoding: mtp @@ -5548,7 +5548,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 2 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp2ep2_hicache_k3_baseline" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -5560,7 +5560,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 2 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp2ep2_hicache_cap48" minimaxm3-fp4-b300-vllm-agentic-mtp: image: vllm/vllm-openai:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 model: nvidia/MiniMax-M3-NVFP4 @@ -8831,7 +8831,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp4_tp4_c4 decode: num-worker: 1 tp: 4 @@ -8849,7 +8849,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp4_tp4_c12 decode: num-worker: 1 tp: 4 @@ -8867,7 +8867,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp4_tp4_c24 decode: num-worker: 1 tp: 4 @@ -8885,7 +8885,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 4 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp4ep4_tp4_c32 decode: num-worker: 1 tp: 4 @@ -8903,7 +8903,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 2 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp2ep2_tp2ep2_c32 decode: num-worker: 1 tp: 2 @@ -8921,7 +8921,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 2 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp2ep2_tp2ep2_c40 decode: num-worker: 1 tp: 2 @@ -8939,7 +8939,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 2 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp2ep2_tp2ep2_c44 decode: num-worker: 1 tp: 2 @@ -8957,7 +8957,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 2 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp2ep2_tp2ep2_c48 decode: num-worker: 1 tp: 2 @@ -8975,7 +8975,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 2 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp2ep2_tp2ep2_c56_replayssm decode: num-worker: 1 tp: 2 From 9df3b3127025f6bdd12a22e80222eafd5cd8c527 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:37:40 -0500 Subject: [PATCH 077/105] docs(recipes): describe AgentX override-variant bundles --- benchmarks/multi_node/srt-slurm-recipes/RECIPES.md | 4 ++-- benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index a731508e05..0800750175 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -15,14 +15,14 @@ Store every recipe at `//-//` block holding only its differences (plain overrides, not `zip_override_*`). Master entries select one with `CONFIG_FILE=recipes//.yaml:override_`. Recipes read as text by a launcher, such as power recipes with top-level `telemetry:`, stay standalone. Keep distinct sweep entry files separate even when their contents match: recipe paths participate in eval grouping. The Qwen3.5 `*-stp-sweep.yaml` and `*-mtp-sweep.yaml` pair preserves that existing distinction. - Update `CONFIG_FILE` and `EVAL_CONFIG_FILE` references in active and deprecated master configs, launcher path rules, workflow filters, and local documentation together when moving a file. Preserve upstream source URLs as provenance and leave historical performance-changelog entries unchanged. No aliases for the old layout are provided. Shared runtime assets stay under `configs/` beside the model directories; they are not standalone recipes. The four files in `configs/dsv4-moe-load-balancer-configs/` are copied verbatim from NVIDIA/srt-slurm commit `deb1dfd9934398664f92d194169c183e009da83b`, preserving the EPLB initial expert assignments used by 17 DSV4 TRT recipes. `setup_srt_slurm()` stages them into the job checkout's `configs/` directory for the recipes' bind mounts. Keeping a recipe in this tree does not activate it; the master configs determine the benchmark matrix. diff --git a/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md b/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md index ef3c0ed034..27cd1c21c3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md +++ b/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md @@ -15,14 +15,14 @@ InferenceX 要求 srt-slurm 2.0 或更新版本，且配置必须声明 `schema: ```text dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml -qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml +qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml ``` - 使用主配置中的 `model-prefix` 和 `precision` 标签。引擎目录为 `sglang`、`vllm`、`trtllm` 或 `tilert`；前端仍在配置内显式声明。硬件目录使用 `b200`、`gb300` 等 GPU 型号，不使用集群名称。 - 工作负载目录为 `1k1k`、`8k1k` 或 `agentx`。已有的跨序列长度配置集合放在 `fixed-seq-len` 下，保留其覆盖项选择器。 - 文件名使用小写字母和连字符，以 `agg` 或 `disagg` 开头。包含拓扑及用于区分同目录配置的关键参数，例如并行方式、批大小、并发数、MTP、卸载或缓存设置。避免日期、带序号的延迟/吞吐量标签，以及重复目录中已有的模型或硬件信息。 - 拓扑名中的 `1p4d` 表示预填充/解码 worker 数，不一定等于物理节点数。`p-tp4` 和 `d-tp8` 分别标识预填充和解码 TP；`b` 表示批大小，`c` 表示并发数。运行参数以 YAML 为准。 -- 覆盖项集合使用 `*-variants.yaml` 命名。即使内容相同，也保留独立扫描入口：配置路径参与评估分组。Qwen3.5 的 `*-stp-sweep.yaml` 和 `*-mtp-sweep.yaml` 保留了这一既有区别。 +- 覆盖项集合使用 `*-variants.yaml` 命名。仅在各配置间存在差异的多节点 AgentX 配置，按主配置条目合并为一个集合，通常为 `agg-variants.yaml` 或 `disagg-variants.yaml`：`base` 保存共享设置，每个原配置成为一个具名 `override_` 块，只包含其差异（使用普通覆盖项，而非 `zip_override_*`）。主配置通过 `CONFIG_FILE=recipes//.yaml:override_` 选择其一。启动器以文本方式读取的配置（例如带顶层 `telemetry:` 的功耗配置）保持独立文件。即使内容相同，也保留独立扫描入口：配置路径参与评估分组。Qwen3.5 的 `*-stp-sweep.yaml` 和 `*-mtp-sweep.yaml` 保留了这一既有区别。 - 移动文件时，同步更新当前及已弃用主配置中的 `CONFIG_FILE`、`EVAL_CONFIG_FILE`，以及启动器路径规则、工作流过滤器和本地文档。保留上游来源 URL，并保持历史性能变更日志不变。不为旧目录结构提供别名。 共享运行时资源保留在模型目录旁的 `configs/` 中，不属于独立基准测试配置。`configs/dsv4-moe-load-balancer-configs/` 中的四个文件原样取自 NVIDIA/srt-slurm 提交 `deb1dfd9934398664f92d194169c183e009da83b`，保留了 17 个 DSV4 TRT 配置使用的 EPLB 初始专家分配。`setup_srt_slurm()` 将这些文件复制到作业仓库的 `configs/` 目录，供配置中的绑定挂载使用。将配置文件放入本目录不会启用该配置；实际基准测试矩阵由主配置决定。 From 94d12a6d61ded1eb664a2f6592d98c4b05ef8040 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:38:57 -0500 Subject: [PATCH 078/105] feat(agentx): port the DSV4 MI355X vLLM AgentX config to srt-slurm The DEP8 points run behind srt-slurm's vLLM Router frontend with consistent-hash session routing, as the legacy script's router did. --- .../configs/pip-runtime-deps.sh | 11 +++ .../dsv4/vllm/mi355x-fp4-mtp/agentic.yaml | 97 +++++++++++++++++++ configs/amd-master.yaml | 4 +- 3 files changed, 110 insertions(+), 2 deletions(-) create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/pip-runtime-deps.sh create mode 100644 benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/pip-runtime-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/pip-runtime-deps.sh new file mode 100755 index 0000000000..a67df2bc04 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/pip-runtime-deps.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Install packages an image lacks before its process starts. The worker role or +# the frontend lists them in SETUP_PIP_PACKAGES; empty installs nothing. +set -euo pipefail +[[ -n "${SETUP_PIP_PACKAGES:-}" ]] || exit 0 +pip_install=(python3 -m pip install --quiet) +if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages; then + pip_install+=(--break-system-packages) +fi +read -r -a packages <<< "$SETUP_PIP_PACKAGES" +"${pip_install[@]}" "${packages[@]}" diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..60220c4a3d --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,97 @@ +# DeepSeek-V4-Pro-0813 AgentX on MI355X with vLLM DSpark K6 +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro?hardware=mi355x). Pure +# TP8 through c64; the wide band runs DP attention (DEP8) behind the vLLM +# Router. The KV cache is GPU-resident. +base: + schema: 2 + name: dsv4-fp4-mi355x-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 + container: vllm/vllm-openai-rocm:nightly-e9757321527ca1ecd514c07c1418dd2c53da3d19 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # 805 GiB checkpoint; cold loads can take the full engine ready timeout. + health_check: + interval_seconds: 10 + max_attempts: 1080 + # The nightly ROCm image lacks some runtime packages; see SETUP_PIP_PACKAGES. + setup_script: pip-runtime-deps.sh + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + # The DSpark checkpoint revision the legacy preflight pinned. + revision: 72e1d3230f6c080a530b0a1d46f8eb4602340597 + tokenizer-revision: 72e1d3230f6c080a530b0a1d46f8eb4602340597 + trust-remote-code: true + async-scheduling: true + distributed-executor-backend: mp + kv-cache-dtype: fp8 + max-num-batched-tokens: 8192 + tensor-parallel-size: 8 + data-parallel-size: 1 + gpu-memory-utilization: 0.86 + moe-backend: aiter + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' + # Six-token DSpark with probabilistic drafting. Throughput runs replace + # verification with the golden acceptance length. + speculative-config: '{"method": "dspark", "num_speculative_tokens": 6, "draft_sample_method": "probabilistic"}' + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + reasoning-parser: deepseek_v4 + enable-auto-tool-choice: true + enable-prefix-caching: true + no-disable-hybrid-kv-cache-manager: true + env: + SETUP_PIP_PACKAGES: Pillow fastapi uvicorn + VLLM_ENGINE_READY_TIMEOUT_S: '10800' + # Sparse SWA prefix-cache tail retention validated for this workload. + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '32768' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_ROCM_USE_AITER: '1' + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 + VLLM_ROCM_USE_AITER_MOE: '1' + # Packed MXFP4 routed experts beside a full-width FP8 shared expert fail + # to load through the fused path. + VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: '0' + # vLLM only clamps torch threads after weight loading. + OMP_NUM_THREADS: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Pure TP admits 2x CONC. DEP8 admits CONC per rank, +# interleaves prefill every 8 steps with a 16k long-prefill threshold, and +# routes each conversation to one rank: the Router hashes X-Session-ID, which +# AIPerf aliases from its stable X-Correlation-ID. +override_tp8_c1: {roles: {agg: {args: {max-num-seqs: 2}}}, benchmark: {env: {CONC: '1'}}} +override_tp8_c4: {roles: {agg: {args: {max-num-seqs: 8}}}, benchmark: {env: {CONC: '4'}}} +override_tp8_c8: {roles: {agg: {args: {max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} +override_tp8_c16: {roles: {agg: {args: {max-num-seqs: 32}}}, benchmark: {env: {CONC: '16'}}} +override_tp8_c32: {roles: {agg: {args: {max-num-seqs: 64}}}, benchmark: {env: {CONC: '32'}}} +override_tp8_c40: {roles: {agg: {args: {max-num-seqs: 80}}}, benchmark: {env: {CONC: '40'}}} +override_tp8_c48: {roles: {agg: {args: {max-num-seqs: 96}}}, benchmark: {env: {CONC: '48'}}} +override_tp8_c56: {roles: {agg: {args: {max-num-seqs: 112}}}, benchmark: {env: {CONC: '56'}}} +override_tp8_c64: {roles: {agg: {args: {max-num-seqs: 128}}}, benchmark: {env: {CONC: '64'}}} +override_dep8_c64: {frontend: {type: vllm-router, args: {policy: consistent_hash, request-timeout-secs: 14400, disable-retries: true}, env: {SETUP_PIP_PACKAGES: 'vllm-router==0.1.14'}}, roles: {agg: {args: {tensor-parallel-size: 1, data-parallel-size: 8, prefill-schedule-interval: 8, long-prefill-token-threshold: 16384, max-num-seqs: 64}}}, benchmark: {env: {CONC: '64', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_dep8_c128: {frontend: {type: vllm-router, args: {policy: consistent_hash, request-timeout-secs: 14400, disable-retries: true}, env: {SETUP_PIP_PACKAGES: 'vllm-router==0.1.14'}}, roles: {agg: {args: {tensor-parallel-size: 1, data-parallel-size: 8, prefill-schedule-interval: 8, long-prefill-token-threshold: 16384, max-num-seqs: 128}}}, benchmark: {env: {CONC: '128', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_dep8_c192: {frontend: {type: vllm-router, args: {policy: consistent_hash, request-timeout-secs: 14400, disable-retries: true}, env: {SETUP_PIP_PACKAGES: 'vllm-router==0.1.14'}}, roles: {agg: {args: {tensor-parallel-size: 1, data-parallel-size: 8, prefill-schedule-interval: 8, long-prefill-token-threshold: 16384, max-num-seqs: 192}}}, benchmark: {env: {CONC: '192', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d27ab6b94b..531e8c1f33 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -904,10 +904,10 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.60 search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32, 40, 48, 56, 64] } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32, 40, 48, 56, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml } # Single-node validation found c128 to be the DPA throughput peak among # c64/c128/c256. Add c192 to locate the knee before the c256 regression. - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64, 128, 192], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64, 128, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml } # LMCache invalid-block recovery currently assumes one KV-cache group, # while speculative decoding creates two. Restore these points after the # upstream hybrid KV recovery fix lands: From 73aa755687a7d77b9cc813ae86d196760b38a8cd Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 11:44:40 -0500 Subject: [PATCH 079/105] fix(agentx): give the MiniMax-M3 Hopper Mooncake master time to install --- .../minimaxm3/vllm/h100-fp8-mtp/agentic.yaml | 8 ++++++-- .../minimaxm3/vllm/h200-fp8-mtp/agentic.yaml | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml index fb6b0f6475..0c47716f94 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml @@ -77,10 +77,12 @@ override_tp8_c6_dram: services: - name: mooncake-master type: mooncake-master + # The in-container install outlasts the default 120 s readiness window. + readiness: {port: 8700, timeout_seconds: 900} preamble: >- pip_install=(python3 -m pip install); python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages && pip_install+=(--break-system-packages); - "${pip_install[@]}" --quiet --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 + "${pip_install[@]}" --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 args: ['--eviction_high_watermark_ratio=0.80', '--eviction_ratio=0.10'] options: store_config: @@ -113,10 +115,12 @@ override_tp8_c8_dram: services: - name: mooncake-master type: mooncake-master + # The in-container install outlasts the default 120 s readiness window. + readiness: {port: 8700, timeout_seconds: 900} preamble: >- pip_install=(python3 -m pip install); python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages && pip_install+=(--break-system-packages); - "${pip_install[@]}" --quiet --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 + "${pip_install[@]}" --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 args: ['--eviction_high_watermark_ratio=0.80', '--eviction_ratio=0.10'] options: store_config: diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml index c0f449c1e7..2d28db4595 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml @@ -79,10 +79,12 @@ override_tp8_c12_dram: services: - name: mooncake-master type: mooncake-master + # The in-container install outlasts the default 120 s readiness window. + readiness: {port: 8700, timeout_seconds: 900} preamble: >- pip_install=(python3 -m pip install); python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages && pip_install+=(--break-system-packages); - "${pip_install[@]}" --quiet --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 + "${pip_install[@]}" --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 args: ['--eviction_high_watermark_ratio=0.80', '--eviction_ratio=0.10'] options: store_config: From 386abc94b4c3965c1685a131896c7801cd8b2e67 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 12:03:11 -0500 Subject: [PATCH 080/105] fix(agentx): give the AMD SGLang AgentX recipes an hour to become healthy The MI300X Qwen3.5 server needed ~27 minutes to load and finish first-request kernel tuning, past srt-slurm's 1800 s default. --- .../glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml | 5 +++++ .../qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml | 5 +++++ .../qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml index 1bc8950316..3af26916f3 100644 --- a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml @@ -18,6 +18,11 @@ base: tachometer: enabled: false engine: sglang + # Loading plus first-request kernel tuning outlasts the 1800 s default: + # MI300X Qwen3.5 was ready after ~27 minutes (run 36160403972). + health_check: + interval_seconds: 10 + max_attempts: 360 roles: agg: nodes: 1 diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml index e9cc55d5fd..a58027c092 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml @@ -18,6 +18,11 @@ base: tachometer: enabled: false engine: sglang + # Loading plus first-request kernel tuning outlasts the 1800 s default: + # MI300X Qwen3.5 was ready after ~27 minutes (run 36160403972). + health_check: + interval_seconds: 10 + max_attempts: 360 roles: agg: nodes: 1 diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml index da992cb3e0..2a36ff79cb 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml @@ -18,6 +18,11 @@ base: tachometer: enabled: false engine: sglang + # Loading plus first-request kernel tuning outlasts the 1800 s default: + # MI300X Qwen3.5 was ready after ~27 minutes (run 36160403972). + health_check: + interval_seconds: 10 + max_attempts: 360 roles: agg: nodes: 1 From 3fe468e18becd8ba3a100806d76da9a85e86e66f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 12:15:41 -0500 Subject: [PATCH 081/105] fix(agentx): extend the DSV4.1 Flash H100 SGLang health window for cold NFS loads --- .../dsv41flash/sglang/h100-fp4-mtp/agentic.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml index ae23f7e092..d551234505 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml @@ -24,6 +24,11 @@ base: tachometer: enabled: false engine: sglang + # Cold loads of the checkpoint and host Engram tables from shared NFS outlast + # the default 1800 s health window (c20 was still loading at 1800 s). + health_check: + interval_seconds: 10 + max_attempts: 360 # Hardware-specific tiling only; checkpoint data, scales and dtypes are unchanged. setup_script: dsv41flash-h100-block32.sh roles: From d5bc165055e5945e7f60d695d2ebc764dd4b9286 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 12:34:33 -0500 Subject: [PATCH 082/105] fix(agentx): extend the MiniMax-M3 H100 vLLM load window for cold NFS loads --- .../minimaxm3/vllm/h100-fp8-mtp/agentic.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml index 0c47716f94..2f8f0f85ab 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml @@ -21,9 +21,11 @@ base: type: vllm connector: null # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + # Lazy loads of the 31 shards from shared NFS ran at ~105 s/shard, past the + # script's 3600 s, so both windows are two hours. health_check: interval_seconds: 10 - max_attempts: 360 + max_attempts: 720 roles: agg: nodes: 1 @@ -52,7 +54,7 @@ base: trust-remote-code: true env: PYTHONNOUSERSITE: '1' - VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_ENGINE_READY_TIMEOUT_S: '7200' benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh From 1e5db281cf9789a89af0b68c2d8dcf60f95875c8 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 12:44:52 -0500 Subject: [PATCH 083/105] fix(h100): keep one uv cache per runner for native single-node jobs --- runners/launch_h100-dgxc-slurm.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 0bbbd84289..2cf9b193cc 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -27,7 +27,8 @@ if [[ "$EXECUTION_PATH" == native-single-node ]]; then SRT_MODEL_PATH="hf:$MODEL" SRT_SQUASH_FILE="/mnt/nfs/lustre/containers/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" # The job's host-side orchestrator cannot write the compute node's home. - export UV_CACHE_DIR="/mnt/nfs/sa-shared/.uv/cache" + # One cache per runner: concurrent builds in a shared NFS cache race. + export UV_CACHE_DIR="/mnt/nfs/sa-shared/.uv/cache-${RUNNER_NAME:?}" export UV_PYTHON_INSTALL_DIR="/mnt/nfs/sa-shared/.uv/python" launch_srt_single_node h100-dgxc-slurm \ --var SLURM_ACCOUNT "$SLURM_ACCOUNT" --var SLURM_PARTITION "$SLURM_PARTITION" \ From 9cb97f9d4e47fa4f87400ef3676133dc6eefbf03 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 12:45:14 -0500 Subject: [PATCH 084/105] fix(b300): give srt-slurm jobs the workflow time limit instead of srtctl's one hour The B300 profile set no default_time_limit, so srtctl submitted native single-node AgentX jobs with --time=01:00:00 and a c32 point timed out. --- runners/launch_b300-dsxe.sh | 2 +- runners/srt-slurm/b300-dsxe.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index ba0269b9ca..e93f81bbf3 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -212,7 +212,7 @@ export OSL="$OSL" SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" echo "Creating srtslurm.yaml configuration..." write_srt_cluster_config b300-dsxe srtslurm.yaml "$USES_DCGM_POWER" \ - --var MODEL_ROOT "$MODEL_ROOT" || exit 1 + --var MODEL_ROOT "$MODEL_ROOT" --var SRT_DEFAULT_TIME_LIMIT "$SALLOC_TIME_LIMIT" || exit 1 echo "Generated srtslurm.yaml:" cat srtslurm.yaml diff --git a/runners/srt-slurm/b300-dsxe.yaml b/runners/srt-slurm/b300-dsxe.yaml index 9da5a6c698..a9ffae66cf 100644 --- a/runners/srt-slurm/b300-dsxe.yaml +++ b/runners/srt-slurm/b300-dsxe.yaml @@ -1,5 +1,7 @@ default_account: ${SLURM_ACCOUNT} default_partition: ${SLURM_PARTITION} +# srtctl otherwise submits with a one-hour limit. +default_time_limit: ${SRT_DEFAULT_TIME_LIMIT} gpus_per_node: 8 network_interface: "" srtctl_root: ${SRTCTL_ROOT} From c673cdc144e2cf4f9d79d728693adae357ed7fbb Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 12:45:30 -0500 Subject: [PATCH 085/105] fix(agentx): report RDMA port states when the Kimi-K3 B300 Mooncake rail probe fails --- .../srt-slurm-recipes/configs/kimik3-b300-mooncake.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh index 5f31c693c4..ed8767556c 100755 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh @@ -22,6 +22,9 @@ for device in mlx5_0 mlx5_1 mlx5_2 mlx5_3 mlx5_4 mlx5_5 mlx5_8 mlx5_9 \ done if [[ -z "$rail" ]]; then echo "Error: no active RDMA rail on $(hostname); Mooncake cannot initialise" >&2 + for state in /sys/class/infiniband/*/ports/*/state; do + echo "$state: $(cat "$state" 2>&1)" >&2 + done exit 1 fi config="${MOONCAKE_CONFIG_PATH:-/logs/mooncake_store_config.json}" From e6382ec04be70278cb8faf2176f6656035bb85c0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 12:50:12 -0500 Subject: [PATCH 086/105] fix(mi355x): let native single-node points use a squash staged on /it-share Node-local /var/lib/squash is not visible from the runner host, so native points always pulled the image, which fails once a nightly tag is pruned. --- runners/launch_mi355x-amds.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 38b15fe1a9..29282f9cc8 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -29,7 +29,8 @@ if [[ "$EXECUTION_PATH" == native-single-node ]]; then export SRT_MODEL_PATH="hf:$MODEL" export SALLOC_TIME_LIMIT=500 export SRT_SRUN_OPTIONS='{"container-remap-root":"", "container-writable":""}' - SRT_SQUASH_FILE="/var/lib/squash/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + # A squash staged on shared storage survives the registry pruning nightly tags. + SRT_SQUASH_FILE="/it-share/gharunners2/srt-slurm/containers/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" launch_srt_single_node mi355x-amds --var GITHUB_WORKSPACE "$GITHUB_WORKSPACE" exit $? fi From 6cbbab5b080ebaff6765e6e2667bf21b337acef4 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 12:58:58 -0500 Subject: [PATCH 087/105] fix(agentx): probe DSXE rdmap RDMA rails for the Kimi-K3 B300 Mooncake store --- .../srt-slurm-recipes/configs/kimik3-b300-mooncake.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh index ed8767556c..243b3d2111 100755 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh @@ -11,10 +11,12 @@ python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null # Rail-isolated nodes: two RNICs cannot reach each other even within a node, so # every rank uses one rail. mlx5_0 is down on some nodes, and topology discovery -# then finds no HCA, so take the first active rail at runtime. +# then finds no HCA, so take the first active rail at runtime. DSXE nodes name +# their rails rdmap*. rail="" for device in mlx5_0 mlx5_1 mlx5_2 mlx5_3 mlx5_4 mlx5_5 mlx5_8 mlx5_9 \ - mlx5_10 mlx5_11 mlx5_16 mlx5_17 mlx5_20 mlx5_21 mlx5_22 mlx5_23; do + mlx5_10 mlx5_11 mlx5_16 mlx5_17 mlx5_20 mlx5_21 mlx5_22 mlx5_23 \ + $(ls /sys/class/infiniband 2>/dev/null | grep '^rdmap' | sort -V); do if grep -q ACTIVE "/sys/class/infiniband/$device/ports/1/state" 2>/dev/null; then rail="$device" break From 968f5e4e1ab6496d336c1f5240cf9e83b0f17709 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 12:59:37 -0500 Subject: [PATCH 088/105] fix(srt): let single-node AgentX evals run without a fixed-sequence context --- benchmarks/single_node/srt_eval.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/srt_eval.sh b/benchmarks/single_node/srt_eval.sh index e6c8af8af4..01525da54a 100644 --- a/benchmarks/single_node/srt_eval.sh +++ b/benchmarks/single_node/srt_eval.sh @@ -10,7 +10,11 @@ SRT_EVAL_STATUS_FILE="$2" trap 'rc=$?; printf "%s\n" "$rc" > "$SRT_EVAL_STATUS_FILE"' EXIT source "$(dirname "${BASH_SOURCE[0]}")/../benchmark_lib.sh" -check_env_vars MODEL MODEL_NAME CONC TP EP_SIZE DP_ATTENTION IS_MULTINODE MAX_MODEL_LEN +check_env_vars MODEL MODEL_NAME CONC TP EP_SIZE DP_ATTENTION IS_MULTINODE +# benchmark_lib clears MAX_MODEL_LEN for AgentX, whose eval uses the native context. +if [[ "${IS_AGENTIC:-0}" != 1 ]]; then + check_env_vars MAX_MODEL_LEN +fi export PORT="${1##*:}" if [[ ! "$PORT" =~ ^[1-9][0-9]*$ || "$IS_MULTINODE" != false ]]; then echo "ERROR: single-node eval requires a local endpoint and single-node metadata" >&2 From c0cc3cc06299c1fc1df71bc540cce32ebb8f2a5a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 13:10:25 -0500 Subject: [PATCH 089/105] fix(srt): evaluate single-node AgentX points with the workflow's framework at native context The single-node post-eval required MAX_MODEL_LEN and forced lm-eval, but benchmark_lib clears MAX_MODEL_LEN for AgentX, so every AgentX eval-only point failed. AgentX now runs run_eval as the multi-node post-eval does. --- benchmarks/single_node/srt_eval.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/srt_eval.sh b/benchmarks/single_node/srt_eval.sh index e6c8af8af4..562ef38e8c 100644 --- a/benchmarks/single_node/srt_eval.sh +++ b/benchmarks/single_node/srt_eval.sh @@ -10,7 +10,13 @@ SRT_EVAL_STATUS_FILE="$2" trap 'rc=$?; printf "%s\n" "$rc" > "$SRT_EVAL_STATUS_FILE"' EXIT source "$(dirname "${BASH_SOURCE[0]}")/../benchmark_lib.sh" -check_env_vars MODEL MODEL_NAME CONC TP EP_SIZE DP_ATTENTION IS_MULTINODE MAX_MODEL_LEN +check_env_vars MODEL MODEL_NAME CONC TP EP_SIZE DP_ATTENTION IS_MULTINODE IS_AGENTIC +# AgentX evaluates at the native context with the workflow's eval framework. +eval_args=() +if [[ "$IS_AGENTIC" != 1 ]]; then + check_env_vars MAX_MODEL_LEN + eval_args=(--framework lm-eval) +fi export PORT="${1##*:}" if [[ ! "$PORT" =~ ^[1-9][0-9]*$ || "$IS_MULTINODE" != false ]]; then echo "ERROR: single-node eval requires a local endpoint and single-node metadata" >&2 @@ -22,6 +28,6 @@ if [[ -d /model ]]; then fi eval_rc=0 -run_eval --framework lm-eval --port "$PORT" || eval_rc=$? +run_eval "${eval_args[@]}" --port "$PORT" || eval_rc=$? append_lm_eval_summary || eval_rc=1 exit "$eval_rc" From ecca6a0a8a19117172387f181c2f86978a2f938f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 13:20:16 -0500 Subject: [PATCH 090/105] fix(b300): download Qwen3.8-Flash-Next NVFP4 into the shared HF cache The checkpoint is staged neither on node-local NVMe nor in the writable model root, so native jobs resolve it by HF id like the other launchers. --- runners/launch_b300-dsxe.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index ff9d011330..f433409ef2 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -139,9 +139,9 @@ if [[ "$EXECUTION_PATH" == native-single-node ]]; then if [[ "$MODEL" == nvidia/DeepSeek-R1-0528-FP4-V2 ]]; then SRT_MODEL_PATH="$MODEL_ROOT/DeepSeek-R1-0528-NVFP4-v2" fi - # Not staged on node-local NVMe; the legacy AgentX path downloaded it here. + # Not staged on node-local NVMe: the engine downloads it into the shared HF cache. if [[ "$MODEL" == RadixArk/Qwen3.8-Flash-Next-NVFP4 ]]; then - SRT_MODEL_PATH="$WRITABLE_MODELS_DIR/${MODEL##*/}" + SRT_MODEL_PATH="hf:$MODEL" fi SRT_SQUASH_FILE="$SQUASH_DIR/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" launch_srt_single_node b300-dsxe \ From 71465e4d25beaf396c9793bc6d8da034d55759d0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 13:27:10 -0500 Subject: [PATCH 091/105] fix(srt): stage single-node AgentX eval artifacts once --- benchmarks/single_node/srt_eval.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/srt_eval.sh b/benchmarks/single_node/srt_eval.sh index 562ef38e8c..33591203ed 100644 --- a/benchmarks/single_node/srt_eval.sh +++ b/benchmarks/single_node/srt_eval.sh @@ -29,5 +29,8 @@ fi eval_rc=0 run_eval "${eval_args[@]}" --port "$PORT" || eval_rc=$? -append_lm_eval_summary || eval_rc=1 +# AgentX eval-only run_eval already staged and removed its results. +if [[ "$IS_AGENTIC" != 1 ]]; then + append_lm_eval_summary || eval_rc=1 +fi exit "$eval_rc" From 6393719194d67b339812b6a8e6407be00235f09d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 13:27:10 -0500 Subject: [PATCH 092/105] fix(srt): stage single-node AgentX eval artifacts once --- benchmarks/single_node/srt_eval.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/srt_eval.sh b/benchmarks/single_node/srt_eval.sh index 562ef38e8c..33591203ed 100644 --- a/benchmarks/single_node/srt_eval.sh +++ b/benchmarks/single_node/srt_eval.sh @@ -29,5 +29,8 @@ fi eval_rc=0 run_eval "${eval_args[@]}" --port "$PORT" || eval_rc=$? -append_lm_eval_summary || eval_rc=1 +# AgentX eval-only run_eval already staged and removed its results. +if [[ "$IS_AGENTIC" != 1 ]]; then + append_lm_eval_summary || eval_rc=1 +fi exit "$eval_rc" From 35b6dbab820a727ec6fdaa37dea5253b662f47ea Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 13:47:32 -0500 Subject: [PATCH 093/105] fix(b300): read models without node-local staging from the shared model root DeepSeek-V4.1-Flash is not in STAGED_MODELS and exists on /scratch only on some nodes, so native points failed wherever it was missing. --- runners/launch_b300-dsxe.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index e93f81bbf3..7478c9e3fc 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -138,6 +138,9 @@ if [[ "$EXECUTION_PATH" == native-single-node ]]; then SRT_MODEL_PATH="$MODEL_ROOT/${MODEL##*/}" if [[ "$MODEL" == nvidia/DeepSeek-R1-0528-FP4-V2 ]]; then SRT_MODEL_PATH="$MODEL_ROOT/DeepSeek-R1-0528-NVFP4-v2" + elif [[ " ${STAGED_MODELS[*]} " != *" ${MODEL##*/} "* || "${MODEL##*/}" == DeepSeek-V4-Pro-0813 ]]; then + # Not staged on every node's NVMe; read the shared copy. + SRT_MODEL_PATH="$SHARED_MODEL_ROOT/${MODEL##*/}" fi SRT_SQUASH_FILE="$SQUASH_DIR/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" launch_srt_single_node b300-dsxe \ From 0a720c52b2d5b92a4ed4a0a6c98ab344f2d57886 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 13:49:04 -0500 Subject: [PATCH 094/105] fix(agentx): give GB200/GB300 DSV4.1 Flash servers a two-hour health budget Cold weight loading from the shared HF cache plus graph capture took just under 30 minutes on GB200 TP2, past srt-slurm's 1800 s default. --- .../dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml | 5 +++++ .../dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml index 95d6ef6eca..7d3a2e810a 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml @@ -44,6 +44,11 @@ base: # Row-sharded host Engram tables free HBM for the 1M-context working set. SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + # Cold weight loads from the shared HF cache plus graph capture outlast + # srt-slurm's 1800 s default, as they did the legacy one-hour deadline. + health_check: + max_attempts: 1440 + interval_seconds: 5 benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml index 9b6b882706..adb716d218 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml @@ -47,6 +47,11 @@ base: SGLANG_DEFAULT_THINKING: '1' SGLANG_DSV41_REASONING_EFFORT: high SGLANG_DSPARK_OPT_MARKOV_W2_BF16: 'True' + # Cold weight loads from the shared HF cache plus graph capture outlast + # srt-slurm's 1800 s default, as they did the legacy one-hour deadline. + health_check: + max_attempts: 1440 + interval_seconds: 5 benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh From 413883d292f019eab306266270b41f0a31c5d511 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 14:06:43 -0500 Subject: [PATCH 095/105] chore(agentx): keep the pruned-image MI355X vLLM configs on their legacy scripts Their nightly images are gone from Docker Hub; the legacy path still runs from node-local squashes. The recipes stay in place for when a squash is staged on /it-share. --- configs/amd-master.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 531e8c1f33..df357bf4b9 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -664,8 +664,8 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.60 search-space: - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1, 4] , spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 10, 12, 14], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1, 4] , spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 10, 12, 14], spec-decoding: mtp } - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [44, 48, 70], spec-decoding: mtp } # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark @@ -1234,8 +1234,8 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.20 search-space: - - { tp: 4, kv-offloading: none, conc-list: [1, 4, 5, 8, 10, 12, 15, 20, 24, 32], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agentic.yaml } - - { tp: 2, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 4, kv-offloading: none, conc-list: [1, 4, 5, 8, 10, 12, 15, 20, 24, 32], spec-decoding: mtp } + - { tp: 2, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via SGLang with MTP speculative # decoding. Two arms: From 51f5c950e076ec197d1a3f83d1d5f8201a796c5f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 14:08:41 -0500 Subject: [PATCH 096/105] fix(agentx): give the DSV4.1 Flash GB200 and GB300 vLLM recipes a two-hour readiness window --- .../dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml | 6 +++--- .../dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml index 6faf35c264..469d8026c9 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml @@ -21,10 +21,10 @@ base: engine: type: vllm connector: null - # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + # Cold loads of the shared-cache checkpoint run past half an hour. health_check: - interval_seconds: 10 - max_attempts: 360 + interval_seconds: 5 + max_attempts: 1440 roles: agg: nodes: 1 diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml index 0f9e3d5c26..806b8555d6 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml @@ -21,10 +21,10 @@ base: engine: type: vllm connector: null - # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + # Cold loads of the shared-cache checkpoint run past half an hour. health_check: - interval_seconds: 10 - max_attempts: 720 + interval_seconds: 5 + max_attempts: 1440 roles: agg: nodes: 1 From 85700c3b0d0a9a5c88810b4c965f901c06797cba Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 14:08:41 -0500 Subject: [PATCH 097/105] fix(agentx): let Mooncake pick the GID on DSXE InfiniBand rails for Kimi-K3 B300 --- .../srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml index d3a1a26731..e2892344b4 100644 --- a/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml @@ -27,7 +27,8 @@ base: max_attempts: 360 # Embedded Mooncake: each TP rank contributes TOTAL_CPU_DRAM_GB / 8 GB. The # setup script pins the client to the master's version and fills in the - # node's active RDMA rail. + # node's active RDMA rail. DSXE rails are InfiniBand without a netdev, so the + # transfer engine picks its own GID (a RoCE v2 index 3 does not exist). setup_script: kimik3-b300-mooncake.sh services: - name: mooncake-master @@ -91,7 +92,6 @@ base: PYTHONHASHSEED: '42' # Outlast AIPerf's pooled-connection idle gaps. VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900' - MC_GID_INDEX: '3' MC_STORE_MEMCPY: '1' MC_ENABLE_DEST_DEVICE_AFFINITY: '1' MC_SLICE_SIZE: '1048576' From 9150c7063220ddf96bdb22adb216c3b8dd15cf01 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 14:12:09 -0500 Subject: [PATCH 098/105] fix(agentx): give the MiniMax-M3 B300 TRT-LLM server a two-hour health budget --- .../minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml index 6ede031d15..b489916ebe 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml @@ -87,6 +87,11 @@ base: PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' + # Piecewise graph capture and autotuning took 22 minutes at TP4 c5; larger + # batch ladders and TP8 outlast srt-slurm's 1800 s default. + health_check: + max_attempts: 1440 + interval_seconds: 5 benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh From 02ca7142e7923a33d0c459616ec0fc076b39b3b3 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 14:21:53 -0500 Subject: [PATCH 099/105] style(agentx): write srt-slurm recipe overrides as block YAML Content is unchanged; flow mappings in override variants become block mappings. vLLM compilation/kv-transfer/speculative configs stay quoted JSON strings, which srtctl passes to vLLM verbatim. --- .../mi355x-fp4/agentx/disagg-umbp-dspark.yaml | 519 +++++++++++++++- .../gb300-fp4/agentx/disagg-variants.yaml | 33 +- .../dsv4/atom/mi355x-fp4-mtp/agentic.yaml | 174 +++++- .../dsv4/sglang/b200-fp4-mtp/agentic.yaml | 342 +++++++++- .../dsv4/sglang/b300-fp4-mtp/agentic.yaml | 148 ++++- .../dsv4/sglang/mi355x-fp4-mtp/agentic.yaml | 162 ++++- .../dsv4/vllm/b200-fp4-mtp/agentic.yaml | 323 +++++++++- .../dsv4/vllm/b300-fp4-mtp/agentic.yaml | 421 ++++++++++++- .../dsv4/vllm/mi355x-fp4-mtp/agentic.yaml | 158 ++++- .../sglang/b200-fp4-mtp/agentic.yaml | 211 ++++++- .../sglang/b300-fp4-mtp/agentic.yaml | 258 +++++++- .../sglang/gb200-fp4-mtp/agentic.yaml | 296 ++++++++- .../sglang/gb300-fp4-mtp/agentic.yaml | 293 ++++++++- .../sglang/h100-fp4-mtp/agentic.yaml | 175 +++++- .../sglang/h200-fp4-mtp/agentic.yaml | 239 ++++++- .../sglang/mi355x-fp4-mtp/agentic.yaml | 36 +- .../dsv41flash/vllm/b200-fp4-mtp/agentic.yaml | 207 +++++- .../dsv41flash/vllm/b300-fp4-mtp/agentic.yaml | 210 ++++++- .../vllm/gb200-fp4-mtp/agentic.yaml | 207 +++++- .../vllm/gb300-fp4-mtp/agentic.yaml | 207 +++++- .../dsv41flash/vllm/h100-fp4-mtp/agentic.yaml | 103 ++- .../dsv41flash/vllm/h200-fp4-mtp/agentic.yaml | 207 +++++- .../vllm/mi300x-fp4-mtp/agentic.yaml | 143 ++++- .../vllm/mi325x-fp4-mtp/agentic.yaml | 233 ++++++- .../vllm/mi355x-fp4-mtp/agentic.yaml | 224 ++++++- .../glm5.2/atom/mi355x-fp4-mtp/agentic.yaml | 90 ++- .../glm5.2/sglang/b200-fp4-mtp/agentic.yaml | 59 +- .../glm5.2/sglang/b200-fp8-mtp/agentic.yaml | 59 +- .../glm5.2/sglang/b300-fp4-mtp/agentic.yaml | 120 +++- .../glm5.2/sglang/b300-fp8-mtp/agentic.yaml | 54 +- .../glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml | 76 ++- .../glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml | 152 ++++- .../glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml | 119 +++- .../kimik3/atom/mi355x-fp4-mtp/agentic.yaml | 19 +- .../kimik3/vllm/b300-fp4-mtp/agentic.yaml | 142 ++++- .../kimik3/vllm/mi355x-fp4-mtp/agentic.yaml | 107 +++- .../atom/mi355x-fp4-mtp/agentic.yaml | 163 ++++- .../trtllm/b200-fp4-mtp/agentic.yaml | 339 +++++++++- .../trtllm/b300-fp4-mtp/agentic.yaml | 342 +++++++++- .../minimaxm3/vllm/b200-fp4-mtp/agentic.yaml | 224 ++++++- .../minimaxm3/vllm/b300-fp4-mtp/agentic.yaml | 105 +++- .../minimaxm3/vllm/h100-fp8-mtp/agentic.yaml | 77 ++- .../minimaxm3/vllm/h200-fp8-mtp/agentic.yaml | 86 ++- .../vllm/mi300x-fp8-mtp/agentic.yaml | 70 ++- .../vllm/mi325x-fp8-mtp/agentic.yaml | 89 ++- .../vllm/mi355x-fp4-mtp/agentic.yaml | 143 ++++- .../qwen3.5/sglang/b200-fp4-mtp/agentic.yaml | 551 +++++++++++++++- .../qwen3.5/sglang/b200-fp8-mtp/agentic.yaml | 552 +++++++++++++++- .../qwen3.5/sglang/b300-fp4-mtp/agentic.yaml | 456 +++++++++++++- .../qwen3.5/sglang/b300-fp8-mtp/agentic.yaml | 588 +++++++++++++++++- .../sglang/mi300x-fp8-mtp/agentic.yaml | 119 +++- .../sglang/mi325x-fp8-mtp/agentic.yaml | 508 ++++++++++++++- .../sglang/mi355x-fp4-mtp/agentic.yaml | 272 +++++++- .../sglang/b200-fp4-mtp/agentic.yaml | 48 +- .../sglang/b300-fp4-mtp/agentic.yaml | 54 +- .../sglang/h200-fp8-mtp/agentic.yaml | 54 +- 56 files changed, 10666 insertions(+), 700 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml index d0402412f9..05ac40f3de 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml @@ -212,20 +212,515 @@ base: # rank under DP attention. DSpark draft length is 6 at c4/c16 (the golden AL # peak) and 3 above; MoRI decode dispatch scales with it (64 x (draft + 1)). override_c4: - roles: {prefill: {gpus: 4, args: {tp-size: 4, mem-fraction-static: 0.86, chunked-prefill-size: 16384, max-running-requests: 8, speculative-dspark-block-size: 6, speculative-num-draft-tokens: 7}}, decode: {gpus: 4, env: {MORI_MAX_DISPATCH_TOKENS_DECODE: "448"}, args: {tp-size: 4, mem-fraction-static: 0.86, max-running-requests: 8, cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8], speculative-dspark-block-size: 6, speculative-num-draft-tokens: 7}}} - benchmark: {env: {CONC: '4'}} + roles: + prefill: + gpus: 4 + args: + tp-size: 4 + mem-fraction-static: 0.86 + chunked-prefill-size: 16384 + max-running-requests: 8 + speculative-dspark-block-size: 6 + speculative-num-draft-tokens: 7 + decode: + gpus: 4 + env: + MORI_MAX_DISPATCH_TOKENS_DECODE: "448" + args: + tp-size: 4 + mem-fraction-static: 0.86 + max-running-requests: 8 + cuda-graph-bs-decode: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + speculative-dspark-block-size: 6 + speculative-num-draft-tokens: 7 + benchmark: + env: + CONC: '4' + override_c16: - roles: {prefill: {gpus: 4, args: {tp-size: 4, mem-fraction-static: 0.86, chunked-prefill-size: 16384, max-running-requests: 32, speculative-dspark-block-size: 6, speculative-num-draft-tokens: 7}}, decode: {gpus: 8, env: {MORI_MAX_DISPATCH_TOKENS_DECODE: "448"}, args: {tp-size: 8, mem-fraction-static: 0.86, max-running-requests: 32, cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], speculative-dspark-block-size: 6, speculative-num-draft-tokens: 7}}} - benchmark: {env: {CONC: '16'}} + roles: + prefill: + gpus: 4 + args: + tp-size: 4 + mem-fraction-static: 0.86 + chunked-prefill-size: 16384 + max-running-requests: 32 + speculative-dspark-block-size: 6 + speculative-num-draft-tokens: 7 + decode: + gpus: 8 + env: + MORI_MAX_DISPATCH_TOKENS_DECODE: "448" + args: + tp-size: 8 + mem-fraction-static: 0.86 + max-running-requests: 32 + cuda-graph-bs-decode: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 + - 22 + - 23 + - 24 + - 25 + - 26 + - 27 + - 28 + - 29 + - 30 + - 31 + - 32 + speculative-dspark-block-size: 6 + speculative-num-draft-tokens: 7 + benchmark: + env: + CONC: '16' + override_c32: - roles: {prefill: {gpus: 4, args: {tp-size: 4, mem-fraction-static: 0.86, chunked-prefill-size: 16384, max-running-requests: 64, speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}, decode: {gpus: 8, env: {MORI_MAX_DISPATCH_TOKENS_DECODE: "256"}, args: {tp-size: 8, mem-fraction-static: 0.86, max-running-requests: 64, cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64], speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}} - benchmark: {env: {CONC: '32'}} + roles: + prefill: + gpus: 4 + args: + tp-size: 4 + mem-fraction-static: 0.86 + chunked-prefill-size: 16384 + max-running-requests: 64 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + decode: + gpus: 8 + env: + MORI_MAX_DISPATCH_TOKENS_DECODE: "256" + args: + tp-size: 8 + mem-fraction-static: 0.86 + max-running-requests: 64 + cuda-graph-bs-decode: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 + - 22 + - 23 + - 24 + - 25 + - 26 + - 27 + - 28 + - 29 + - 30 + - 31 + - 32 + - 33 + - 34 + - 35 + - 36 + - 37 + - 38 + - 39 + - 40 + - 41 + - 42 + - 43 + - 44 + - 45 + - 46 + - 47 + - 48 + - 49 + - 50 + - 51 + - 52 + - 53 + - 54 + - 55 + - 56 + - 57 + - 58 + - 59 + - 60 + - 61 + - 62 + - 63 + - 64 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + benchmark: + env: + CONC: '32' + override_c48: - roles: {prefill: {gpus: 4, args: {tp-size: 4, mem-fraction-static: 0.86, chunked-prefill-size: 16384, max-running-requests: 96, speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}, decode: {gpus: 8, env: {MORI_MAX_DISPATCH_TOKENS_DECODE: "256"}, args: {tp-size: 8, mem-fraction-static: 0.86, max-running-requests: 96, cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96], speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}} - benchmark: {env: {CONC: '48'}} + roles: + prefill: + gpus: 4 + args: + tp-size: 4 + mem-fraction-static: 0.86 + chunked-prefill-size: 16384 + max-running-requests: 96 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + decode: + gpus: 8 + env: + MORI_MAX_DISPATCH_TOKENS_DECODE: "256" + args: + tp-size: 8 + mem-fraction-static: 0.86 + max-running-requests: 96 + cuda-graph-bs-decode: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 + - 22 + - 23 + - 24 + - 25 + - 26 + - 27 + - 28 + - 29 + - 30 + - 31 + - 32 + - 33 + - 34 + - 35 + - 36 + - 37 + - 38 + - 39 + - 40 + - 41 + - 42 + - 43 + - 44 + - 45 + - 46 + - 47 + - 48 + - 49 + - 50 + - 51 + - 52 + - 53 + - 54 + - 55 + - 56 + - 57 + - 58 + - 59 + - 60 + - 61 + - 62 + - 63 + - 64 + - 65 + - 66 + - 67 + - 68 + - 69 + - 70 + - 71 + - 72 + - 73 + - 74 + - 75 + - 76 + - 77 + - 78 + - 79 + - 80 + - 81 + - 82 + - 83 + - 84 + - 85 + - 86 + - 87 + - 88 + - 89 + - 90 + - 91 + - 92 + - 93 + - 94 + - 95 + - 96 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + benchmark: + env: + CONC: '48' + override_c192: - roles: {prefill: {gpus: 8, env: {GPU_MAX_HW_QUEUES: "5", SGLANG_SHARED_EXPERT_TP1: "1", SGLANG_DP_SHARED_EXPERT_LOCAL: "1", SGLANG_DP_USE_GATHERV: "1", SGLANG_DP_USE_REDUCE_SCATTER: "1", }, args: {tp-size: 8, dp-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, swa-full-tokens-ratio: 0.15, mem-fraction-static: 0.92, chunked-prefill-size: 65536, enable-two-batch-overlap: false, max-running-requests: 384, speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}, decode: {gpus: 8, env: {GPU_MAX_HW_QUEUES: "5", SGLANG_SHARED_EXPERT_TP1: "1", SGLANG_DP_SHARED_EXPERT_LOCAL: "1", SGLANG_DP_USE_GATHERV: "1", SGLANG_DP_USE_REDUCE_SCATTER: "1", MORI_MAX_DISPATCH_TOKENS_DECODE: "256"}, args: {tp-size: 8, dp-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, swa-full-tokens-ratio: 0.15, mem-fraction-static: 0.92, max-running-requests: 384, cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48], speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}} - benchmark: {env: {CONC: '192', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'}} + roles: + prefill: + gpus: 8 + env: + GPU_MAX_HW_QUEUES: "5" + SGLANG_SHARED_EXPERT_TP1: "1" + SGLANG_DP_SHARED_EXPERT_LOCAL: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_DP_USE_REDUCE_SCATTER: "1" + args: + tp-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + swa-full-tokens-ratio: 0.15 + mem-fraction-static: 0.92 + chunked-prefill-size: 65536 + enable-two-batch-overlap: false + max-running-requests: 384 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + decode: + gpus: 8 + env: + GPU_MAX_HW_QUEUES: "5" + SGLANG_SHARED_EXPERT_TP1: "1" + SGLANG_DP_SHARED_EXPERT_LOCAL: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_DP_USE_REDUCE_SCATTER: "1" + MORI_MAX_DISPATCH_TOKENS_DECODE: "256" + args: + tp-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + swa-full-tokens-ratio: 0.15 + mem-fraction-static: 0.92 + max-running-requests: 384 + cuda-graph-bs-decode: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 + - 22 + - 23 + - 24 + - 25 + - 26 + - 27 + - 28 + - 29 + - 30 + - 31 + - 32 + - 33 + - 34 + - 35 + - 36 + - 37 + - 38 + - 39 + - 40 + - 41 + - 42 + - 43 + - 44 + - 45 + - 46 + - 47 + - 48 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + benchmark: + env: + CONC: '192' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + override_c256: - roles: {prefill: {gpus: 8, env: {GPU_MAX_HW_QUEUES: "5", SGLANG_SHARED_EXPERT_TP1: "1", SGLANG_DP_SHARED_EXPERT_LOCAL: "1", SGLANG_DP_USE_GATHERV: "1", SGLANG_DP_USE_REDUCE_SCATTER: "1", }, args: {tp-size: 8, dp-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, swa-full-tokens-ratio: 0.15, mem-fraction-static: 0.92, chunked-prefill-size: 65536, enable-two-batch-overlap: false, max-running-requests: 512, speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}, decode: {gpus: 8, env: {GPU_MAX_HW_QUEUES: "5", SGLANG_SHARED_EXPERT_TP1: "1", SGLANG_DP_SHARED_EXPERT_LOCAL: "1", SGLANG_DP_USE_GATHERV: "1", SGLANG_DP_USE_REDUCE_SCATTER: "1", MORI_MAX_DISPATCH_TOKENS_DECODE: "256"}, args: {tp-size: 8, dp-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, swa-full-tokens-ratio: 0.15, mem-fraction-static: 0.92, max-running-requests: 512, cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64], speculative-dspark-block-size: 3, speculative-num-draft-tokens: 4}}} - benchmark: {env: {CONC: '256', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'}} + roles: + prefill: + gpus: 8 + env: + GPU_MAX_HW_QUEUES: "5" + SGLANG_SHARED_EXPERT_TP1: "1" + SGLANG_DP_SHARED_EXPERT_LOCAL: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_DP_USE_REDUCE_SCATTER: "1" + args: + tp-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + swa-full-tokens-ratio: 0.15 + mem-fraction-static: 0.92 + chunked-prefill-size: 65536 + enable-two-batch-overlap: false + max-running-requests: 512 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + decode: + gpus: 8 + env: + GPU_MAX_HW_QUEUES: "5" + SGLANG_SHARED_EXPERT_TP1: "1" + SGLANG_DP_SHARED_EXPERT_LOCAL: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_DP_USE_REDUCE_SCATTER: "1" + MORI_MAX_DISPATCH_TOKENS_DECODE: "256" + args: + tp-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + swa-full-tokens-ratio: 0.15 + mem-fraction-static: 0.92 + max-running-requests: 512 + cuda-graph-bs-decode: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 + - 22 + - 23 + - 24 + - 25 + - 26 + - 27 + - 28 + - 29 + - 30 + - 31 + - 32 + - 33 + - 34 + - 35 + - 36 + - 37 + - 38 + - 39 + - 40 + - 41 + - 42 + - 43 + - 44 + - 45 + - 46 + - 47 + - 48 + - 49 + - 50 + - 51 + - 52 + - 53 + - 54 + - 55 + - 56 + - 57 + - 58 + - 59 + - 60 + - 61 + - 62 + - 63 + - 64 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + benchmark: + env: + CONC: '256' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml index 1cefab7def..c8e5cfd844 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -213,7 +213,8 @@ override_1p1d_tp8_c1_b1_mtp5: gpus: 8 args: cuda_graph_config: - batch_sizes: [1] + batch_sizes: + - 1 max_batch_size: 1 moe_expert_parallel_size: 1 sparse_attention_config: @@ -237,7 +238,11 @@ override_1p1d_tep8_c20_b5_mtp5: gpus: 8 args: cuda_graph_config: - batch_sizes: [1, 2, 4, 5] + batch_sizes: + - 1 + - 2 + - 4 + - 5 max_batch_size: 5 moe_expert_parallel_size: 8 enable_iter_perf_stats: true @@ -266,7 +271,9 @@ override_1p4d_tep4_c30_b2_mtp5: gpus: 4 args: cuda_graph_config: - batch_sizes: [1, 2] + batch_sizes: + - 1 + - 2 max_batch_size: 2 moe_expert_parallel_size: 4 enable_iter_perf_stats: true @@ -295,7 +302,11 @@ override_3p4d_tep4_c60_b5_mtp5: gpus: 4 args: cuda_graph_config: - batch_sizes: [1, 2, 4, 5] + batch_sizes: + - 1 + - 2 + - 4 + - 5 max_batch_size: 5 moe_expert_parallel_size: 4 enable_iter_perf_stats: true @@ -325,7 +336,12 @@ override_4p1d_dep8_c227_b16_mtp3: gpus: 8 args: cuda_graph_config: - batch_sizes: [1, 2, 4, 8, 16] + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 enable_attention_dp: true enable_lm_head_tp_in_adp: false kv_cache_config: @@ -362,7 +378,12 @@ override_5p1d_dep16_c260_b16_mtp3: gpus: 16 args: cuda_graph_config: - batch_sizes: [1, 2, 4, 8, 16] + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 enable_attention_dp: true enable_lm_head_tp_in_adp: false kv_cache_config: diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml index e7d4f74475..49bdd63b51 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml @@ -61,13 +61,167 @@ base: # One variant per point. Admission is 2x CONC. KV is BF16 through concurrency 16 # and FP8 in the DEP band. DEP pins sessions to one DP-attention rank so later # turns reach their prefix KV, and captures every small per-rank decode batch. -override_tp8_c1: {roles: {agg: {args: {max-num-seqs: 2, kv-cache-dtype: bf16}}}, benchmark: {env: {CONC: '1'}}} -override_tp8_c2: {roles: {agg: {args: {max-num-seqs: 4, kv-cache-dtype: bf16}}}, benchmark: {env: {CONC: '2'}}} -override_tp8_c4: {roles: {agg: {args: {max-num-seqs: 8, kv-cache-dtype: bf16}}}, benchmark: {env: {CONC: '4'}}} -override_tp8_c8: {roles: {agg: {args: {max-num-seqs: 16, kv-cache-dtype: bf16}}}, benchmark: {env: {CONC: '8'}}} -override_tp8_c16: {roles: {agg: {args: {max-num-seqs: 32, kv-cache-dtype: bf16}}}, benchmark: {env: {CONC: '16'}}} -override_tp8_c48_dep: {roles: {agg: {args: {max-num-seqs: 96, kv-cache-dtype: fp8, enable-expert-parallel: true, enable-dp-attention: true, all2all-backend: rccl, dp-load-balance: least_tokens, moe-backend: standard, cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]'}, env: {ATOM_DP_SESSION_AFFINITY: '1', ATOM_DP_LB_REQ_EQUIV: '512', ATOM_ENABLE_PREFILL_DELAYER: '1', ATOM_PREFILL_DECODE_INTERVAL: '10'}}}, benchmark: {env: {CONC: '48', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp8_c64_dep: {roles: {agg: {args: {max-num-seqs: 128, kv-cache-dtype: fp8, enable-expert-parallel: true, enable-dp-attention: true, all2all-backend: rccl, dp-load-balance: least_tokens, moe-backend: standard, cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]'}, env: {ATOM_DP_SESSION_AFFINITY: '1', ATOM_DP_LB_REQ_EQUIV: '512', ATOM_ENABLE_PREFILL_DELAYER: '1', ATOM_PREFILL_DECODE_INTERVAL: '10'}}}, benchmark: {env: {CONC: '64', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp8_c96_dep: {roles: {agg: {args: {max-num-seqs: 192, kv-cache-dtype: fp8, enable-expert-parallel: true, enable-dp-attention: true, all2all-backend: rccl, dp-load-balance: least_tokens, moe-backend: standard, cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]'}, env: {ATOM_DP_SESSION_AFFINITY: '1', ATOM_DP_LB_REQ_EQUIV: '512', ATOM_ENABLE_PREFILL_DELAYER: '1', ATOM_PREFILL_DECODE_INTERVAL: '10'}}}, benchmark: {env: {CONC: '96', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp8_c128_dep: {roles: {agg: {args: {max-num-seqs: 256, kv-cache-dtype: fp8, enable-expert-parallel: true, enable-dp-attention: true, all2all-backend: rccl, dp-load-balance: least_tokens, moe-backend: standard, cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]'}, env: {ATOM_DP_SESSION_AFFINITY: '1', ATOM_DP_LB_REQ_EQUIV: '512', ATOM_ENABLE_PREFILL_DELAYER: '1', ATOM_PREFILL_DECODE_INTERVAL: '10'}}}, benchmark: {env: {CONC: '128', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp8_c256_dep: {roles: {agg: {args: {max-num-seqs: 512, kv-cache-dtype: fp8, enable-expert-parallel: true, enable-dp-attention: true, all2all-backend: rccl, dp-load-balance: least_tokens, moe-backend: standard, cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]'}, env: {ATOM_DP_SESSION_AFFINITY: '1', ATOM_DP_LB_REQ_EQUIV: '512', ATOM_ENABLE_PREFILL_DELAYER: '1', ATOM_PREFILL_DECODE_INTERVAL: '10'}}}, benchmark: {env: {CONC: '256', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp8_c1: + roles: + agg: + args: + max-num-seqs: 2 + kv-cache-dtype: bf16 + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + args: + max-num-seqs: 4 + kv-cache-dtype: bf16 + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + args: + max-num-seqs: 8 + kv-cache-dtype: bf16 + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + args: + max-num-seqs: 16 + kv-cache-dtype: bf16 + benchmark: + env: + CONC: '8' + +override_tp8_c16: + roles: + agg: + args: + max-num-seqs: 32 + kv-cache-dtype: bf16 + benchmark: + env: + CONC: '16' + +override_tp8_c48_dep: + roles: + agg: + args: + max-num-seqs: 96 + kv-cache-dtype: fp8 + enable-expert-parallel: true + enable-dp-attention: true + all2all-backend: rccl + dp-load-balance: least_tokens + moe-backend: standard + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]' + env: + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '10' + benchmark: + env: + CONC: '48' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c64_dep: + roles: + agg: + args: + max-num-seqs: 128 + kv-cache-dtype: fp8 + enable-expert-parallel: true + enable-dp-attention: true + all2all-backend: rccl + dp-load-balance: least_tokens + moe-backend: standard + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]' + env: + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '10' + benchmark: + env: + CONC: '64' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c96_dep: + roles: + agg: + args: + max-num-seqs: 192 + kv-cache-dtype: fp8 + enable-expert-parallel: true + enable-dp-attention: true + all2all-backend: rccl + dp-load-balance: least_tokens + moe-backend: standard + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]' + env: + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '10' + benchmark: + env: + CONC: '96' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c128_dep: + roles: + agg: + args: + max-num-seqs: 256 + kv-cache-dtype: fp8 + enable-expert-parallel: true + enable-dp-attention: true + all2all-backend: rccl + dp-load-balance: least_tokens + moe-backend: standard + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]' + env: + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '10' + benchmark: + env: + CONC: '128' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c256_dep: + roles: + agg: + args: + max-num-seqs: 512 + kv-cache-dtype: fp8 + enable-expert-parallel: true + enable-dp-attention: true + all2all-backend: rccl + dp-load-balance: least_tokens + moe-backend: standard + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]' + env: + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '10' + benchmark: + env: + CONC: '256' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml index 5f317e6ebf..36bba702cb 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml @@ -76,15 +76,333 @@ base: # DP rank, and balances DP admission; the gateway routes sessions by AIPerf # correlation ID. C160 interleaves prefill more often and bounds the gateway's # load imbalance. -override_tp8_c1: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 2, cuda-graph-max-bs: 2}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp8_c2: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 4, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '2', KV_OFFLOADING: none}}} -override_tp8_c3: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 6, cuda-graph-max-bs: 6}}}, benchmark: {env: {CONC: '3', KV_OFFLOADING: none}}} -override_tp8_c4: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 8, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} -override_tp8_c5: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 10, cuda-graph-max-bs: 10}}}, benchmark: {env: {CONC: '5', KV_OFFLOADING: none}}} -override_tp8_hicache_c8: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 16, cuda-graph-max-bs: 16, enable-hierarchical-cache: true, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, hicache-ratio: 2.75}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} -override_tp8_hicache_c10: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 20, cuda-graph-max-bs: 20, enable-hierarchical-cache: true, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, hicache-ratio: 2.75}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} -override_tp8_hicache_c16: {roles: {agg: {args: {moe-runner-backend: flashinfer_mxfp4, mem-fraction-static: 0.9, swa-full-tokens-ratio: 0.1, chunked-prefill-size: 8192, max-running-requests: 32, cuda-graph-max-bs: 32, enable-hierarchical-cache: true, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, hicache-ratio: 2.75}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} -override_dep8_hicache_c64: {frontend: {type: sglang-router, args: {policy: cache_aware, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, expert-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, incremental-streaming-output: true, stream-interval: 20, tokenizer-worker-num: 8, load-balance-method: total_requests, load-snapshot-publish-interval: 1, prefill-decode-interval: 24, moe-a2a-backend: megamoe, enable-w4a4-mxfp4-megamoe: true, disable-shared-experts-fusion: true, mem-fraction-static: 0.88, swa-full-tokens-ratio: 0.02, chunked-prefill-size: 49152, max-running-requests: 128, cuda-graph-max-bs: 32, enable-hierarchical-cache: true, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, hicache-ratio: 8}, env: {SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320'}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'}}} -override_dep8_hicache_c96: {frontend: {type: sglang-router, args: {policy: cache_aware, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, expert-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, incremental-streaming-output: true, stream-interval: 20, tokenizer-worker-num: 8, load-balance-method: total_requests, load-snapshot-publish-interval: 1, prefill-decode-interval: 24, moe-a2a-backend: megamoe, enable-w4a4-mxfp4-megamoe: true, disable-shared-experts-fusion: true, mem-fraction-static: 0.88, swa-full-tokens-ratio: 0.02, chunked-prefill-size: 49152, max-running-requests: 192, cuda-graph-max-bs: 32, enable-hierarchical-cache: true, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, hicache-ratio: 8}, env: {SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320'}}}, benchmark: {env: {CONC: '96', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'}}} -override_dep8_hicache_c128: {frontend: {type: sglang-router, args: {policy: cache_aware, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, expert-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, incremental-streaming-output: true, stream-interval: 20, tokenizer-worker-num: 8, load-balance-method: total_requests, load-snapshot-publish-interval: 1, prefill-decode-interval: 24, moe-a2a-backend: megamoe, enable-w4a4-mxfp4-megamoe: true, disable-shared-experts-fusion: true, mem-fraction-static: 0.88, swa-full-tokens-ratio: 0.02, chunked-prefill-size: 49152, max-running-requests: 256, cuda-graph-max-bs: 32, enable-hierarchical-cache: true, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, hicache-ratio: 8}, env: {SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320'}}}, benchmark: {env: {CONC: '128', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'}}} -override_dep8_hicache_c160: {frontend: {type: sglang-router, args: {policy: cache_aware, balance-abs-threshold: 32, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, expert-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-dp-attention-local-control-broadcast: true, incremental-streaming-output: true, stream-interval: 20, tokenizer-worker-num: 8, load-balance-method: total_requests, load-snapshot-publish-interval: 1, prefill-decode-interval: 20, moe-a2a-backend: megamoe, enable-w4a4-mxfp4-megamoe: true, disable-shared-experts-fusion: true, mem-fraction-static: 0.88, swa-full-tokens-ratio: 0.02, chunked-prefill-size: 49152, max-running-requests: 320, cuda-graph-max-bs: 32, enable-hierarchical-cache: true, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, hicache-ratio: 8}, env: {SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320'}}}, benchmark: {env: {CONC: '160', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true', AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'}}} +override_tp8_c1: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 2 + cuda-graph-max-bs: 2 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c2: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 4 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + +override_tp8_c3: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 6 + cuda-graph-max-bs: 6 + benchmark: + env: + CONC: '3' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 8 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c5: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 10 + cuda-graph-max-bs: 10 + benchmark: + env: + CONC: '5' + KV_OFFLOADING: none + +override_tp8_hicache_c8: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 16 + cuda-graph-max-bs: 16 + enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 2.75 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_hicache_c10: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 20 + cuda-graph-max-bs: 20 + enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 2.75 + benchmark: + env: + CONC: '10' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_hicache_c16: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 32 + cuda-graph-max-bs: 32 + enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 2.75 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_dep8_hicache_c64: + frontend: + type: sglang-router + args: + policy: cache_aware + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + incremental-streaming-output: true + stream-interval: 20 + tokenizer-worker-num: 8 + load-balance-method: total_requests + load-snapshot-publish-interval: 1 + prefill-decode-interval: 24 + moe-a2a-backend: megamoe + enable-w4a4-mxfp4-megamoe: true + disable-shared-experts-fusion: true + mem-fraction-static: 0.88 + swa-full-tokens-ratio: 0.02 + chunked-prefill-size: 49152 + max-running-requests: 128 + cuda-graph-max-bs: 32 + enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 8 + env: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320' + benchmark: + env: + CONC: '64' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + +override_dep8_hicache_c96: + frontend: + type: sglang-router + args: + policy: cache_aware + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + incremental-streaming-output: true + stream-interval: 20 + tokenizer-worker-num: 8 + load-balance-method: total_requests + load-snapshot-publish-interval: 1 + prefill-decode-interval: 24 + moe-a2a-backend: megamoe + enable-w4a4-mxfp4-megamoe: true + disable-shared-experts-fusion: true + mem-fraction-static: 0.88 + swa-full-tokens-ratio: 0.02 + chunked-prefill-size: 49152 + max-running-requests: 192 + cuda-graph-max-bs: 32 + enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 8 + env: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320' + benchmark: + env: + CONC: '96' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + +override_dep8_hicache_c128: + frontend: + type: sglang-router + args: + policy: cache_aware + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + incremental-streaming-output: true + stream-interval: 20 + tokenizer-worker-num: 8 + load-balance-method: total_requests + load-snapshot-publish-interval: 1 + prefill-decode-interval: 24 + moe-a2a-backend: megamoe + enable-w4a4-mxfp4-megamoe: true + disable-shared-experts-fusion: true + mem-fraction-static: 0.88 + swa-full-tokens-ratio: 0.02 + chunked-prefill-size: 49152 + max-running-requests: 256 + cuda-graph-max-bs: 32 + enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 8 + env: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320' + benchmark: + env: + CONC: '128' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + +override_dep8_hicache_c160: + frontend: + type: sglang-router + args: + policy: cache_aware + balance-abs-threshold: 32 + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + incremental-streaming-output: true + stream-interval: 20 + tokenizer-worker-num: 8 + load-balance-method: total_requests + load-snapshot-publish-interval: 1 + prefill-decode-interval: 20 + moe-a2a-backend: megamoe + enable-w4a4-mxfp4-megamoe: true + disable-shared-experts-fusion: true + mem-fraction-static: 0.88 + swa-full-tokens-ratio: 0.02 + chunked-prefill-size: 49152 + max-running-requests: 320 + cuda-graph-max-bs: 32 + enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 8 + env: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320' + benchmark: + env: + CONC: '160' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml index c7618db209..3beb2ef09c 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml @@ -74,11 +74,60 @@ base: # One variant per point. TP8: admission is 2x CONC and the graph batch 4x # CONC, capped at 64. -override_tp8_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp8_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} -override_tp8_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} -override_tp8_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: none}}} -override_tp8_c32: {roles: {agg: {args: {max-running-requests: 64, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: none}}} +override_tp8_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp8_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp8_c32: + roles: + agg: + args: + max-running-requests: 64 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: none # DEP8 replaces the TP8 MoE path: the router keeps each session on the DP rank # holding its radix prefix, the global prefill chunk is 8192 per rank, decode @@ -140,9 +189,86 @@ override_dep8_c32: KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '2849' CONC: '32' -override_dep8_c64: {frontend: *dep8_frontend, roles: {agg: {args: {<<: *dep8_args, max-running-requests: 128, mem-fraction-static: 0.9}, env: *dep8_env}}, benchmark: {env: {<<: *dep8_client, CONC: '64'}}} -override_dep8_c128: {frontend: *dep8_frontend, roles: {agg: {args: {<<: *dep8_args, max-running-requests: 256, mem-fraction-static: 0.9}, env: *dep8_env}}, benchmark: {env: {<<: *dep8_client, CONC: '128'}}} -override_dep8_c256: {frontend: *dep8_frontend, roles: {agg: {args: {<<: *dep8_args, max-running-requests: 512, mem-fraction-static: 0.9}, env: *dep8_env}}, benchmark: {env: {<<: *dep8_client, CONC: '256'}}} -override_dep8_c384: {frontend: *dep8_frontend, roles: {agg: {args: {<<: *dep8_args, max-running-requests: 768, mem-fraction-static: 0.88}, env: *dep8_env}}, benchmark: {env: {<<: *dep8_client, CONC: '384'}}} -override_dep8_c512: {frontend: *dep8_frontend, roles: {agg: {args: {<<: *dep8_args, max-running-requests: 1024, mem-fraction-static: 0.86}, env: *dep8_env}}, benchmark: {env: {<<: *dep8_client, CONC: '512'}}} -override_dep8_c576: {frontend: *dep8_frontend, roles: {agg: {args: {<<: *dep8_args, max-running-requests: 1152, mem-fraction-static: 0.86}, env: *dep8_env}}, benchmark: {env: {<<: *dep8_client, CONC: '576'}}} +override_dep8_c64: + frontend: *dep8_frontend + roles: + agg: + args: + <<: *dep8_args + max-running-requests: 128 + mem-fraction-static: 0.9 + env: *dep8_env + benchmark: + env: + <<: *dep8_client + CONC: '64' + +override_dep8_c128: + frontend: *dep8_frontend + roles: + agg: + args: + <<: *dep8_args + max-running-requests: 256 + mem-fraction-static: 0.9 + env: *dep8_env + benchmark: + env: + <<: *dep8_client + CONC: '128' + +override_dep8_c256: + frontend: *dep8_frontend + roles: + agg: + args: + <<: *dep8_args + max-running-requests: 512 + mem-fraction-static: 0.9 + env: *dep8_env + benchmark: + env: + <<: *dep8_client + CONC: '256' + +override_dep8_c384: + frontend: *dep8_frontend + roles: + agg: + args: + <<: *dep8_args + max-running-requests: 768 + mem-fraction-static: 0.88 + env: *dep8_env + benchmark: + env: + <<: *dep8_client + CONC: '384' + +override_dep8_c512: + frontend: *dep8_frontend + roles: + agg: + args: + <<: *dep8_args + max-running-requests: 1024 + mem-fraction-static: 0.86 + env: *dep8_env + benchmark: + env: + <<: *dep8_client + CONC: '512' + +override_dep8_c576: + frontend: *dep8_frontend + roles: + agg: + args: + <<: *dep8_args + max-running-requests: 1152 + mem-fraction-static: 0.86 + env: *dep8_env + benchmark: + env: + <<: *dep8_client + CONC: '576' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml index 8405cf812c..b3aacda995 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml @@ -85,10 +85,158 @@ base: # up to 128. HiCache pins 1.5x the device KV pool per rank. DP attention keeps # 8192 prefill tokens per rank, and the router pins each AIPerf session to the # rank holding its prefix. Saturation points get a longer warmup drain. -override_tp8_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs-decode: 2}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp8_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs-decode: 8}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} -override_tp8_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs-decode: 32}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: none}}} -override_tp8_c32_hicache: {roles: {agg: {args: {max-running-requests: 64, cuda-graph-max-bs-decode: 64, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: dram, AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp8_c48_hicache: {roles: {agg: {args: {max-running-requests: 96, cuda-graph-max-bs-decode: 96, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct}}}, benchmark: {env: {CONC: '48', KV_OFFLOADING: dram, AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp8_c128_dp_hicache: {frontend: {type: sglang-router, args: {policy: cache_aware, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {max-running-requests: 256, cuda-graph-max-bs-decode: 128, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, data-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-prefill-delayer: true, enable-dp-attention-local-control-broadcast: true, tokenizer-worker-num: 8, stream-interval: 20, prefill-delayer-token-usage-low-watermark: 0.7, chunked-prefill-size: 65536, mem-fraction-static: 0.92}, env: {SGLANG_SHARED_EXPERT_TP1: '1', SGLANG_DP_SHARED_EXPERT_LOCAL: '1', SGLANG_DP_USE_GATHERV: '1', SGLANG_DP_USE_REDUCE_SCATTER: '1', GPU_MAX_HW_QUEUES: '5'}}}, benchmark: {env: {CONC: '128', KV_OFFLOADING: dram, AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp8_c256_dp_hicache: {frontend: {type: sglang-router, args: {policy: cache_aware, balance-abs-threshold: 32, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {max-running-requests: 512, cuda-graph-max-bs-decode: 128, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: direct, hicache-mem-layout: page_first_direct, data-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, enable-prefill-delayer: true, enable-dp-attention-local-control-broadcast: true, tokenizer-worker-num: 8, stream-interval: 20, prefill-delayer-token-usage-low-watermark: 0.7, chunked-prefill-size: 65536, mem-fraction-static: 0.92}, env: {SGLANG_SHARED_EXPERT_TP1: '1', SGLANG_DP_SHARED_EXPERT_LOCAL: '1', SGLANG_DP_USE_GATHERV: '1', SGLANG_DP_USE_REDUCE_SCATTER: '1', GPU_MAX_HW_QUEUES: '5'}}}, benchmark: {env: {CONC: '256', KV_OFFLOADING: dram, AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp8_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp8_c32_hicache: + roles: + agg: + args: + max-running-requests: 64 + cuda-graph-max-bs-decode: 64 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c48_hicache: + roles: + agg: + args: + max-running-requests: 96 + cuda-graph-max-bs-decode: 96 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + benchmark: + env: + CONC: '48' + KV_OFFLOADING: dram + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c128_dp_hicache: + frontend: + type: sglang-router + args: + policy: cache_aware + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + max-running-requests: 256 + cuda-graph-max-bs-decode: 128 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-prefill-delayer: true + enable-dp-attention-local-control-broadcast: true + tokenizer-worker-num: 8 + stream-interval: 20 + prefill-delayer-token-usage-low-watermark: 0.7 + chunked-prefill-size: 65536 + mem-fraction-static: 0.92 + env: + SGLANG_SHARED_EXPERT_TP1: '1' + SGLANG_DP_SHARED_EXPERT_LOCAL: '1' + SGLANG_DP_USE_GATHERV: '1' + SGLANG_DP_USE_REDUCE_SCATTER: '1' + GPU_MAX_HW_QUEUES: '5' + benchmark: + env: + CONC: '128' + KV_OFFLOADING: dram + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c256_dp_hicache: + frontend: + type: sglang-router + args: + policy: cache_aware + balance-abs-threshold: 32 + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + max-running-requests: 512 + cuda-graph-max-bs-decode: 128 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-prefill-delayer: true + enable-dp-attention-local-control-broadcast: true + tokenizer-worker-num: 8 + stream-interval: 20 + prefill-delayer-token-usage-low-watermark: 0.7 + chunked-prefill-size: 65536 + mem-fraction-static: 0.92 + env: + SGLANG_SHARED_EXPERT_TP1: '1' + SGLANG_DP_SHARED_EXPERT_LOCAL: '1' + SGLANG_DP_USE_GATHERV: '1' + SGLANG_DP_USE_REDUCE_SCATTER: '1' + GPU_MAX_HW_QUEUES: '5' + benchmark: + env: + CONC: '256' + KV_OFFLOADING: dram + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml index 36de08edc1..44c6285845 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml @@ -73,15 +73,314 @@ base: # 2x CONC across the ranks with MegaMoE experts, reserves verification slots # beyond an 8192-token prefill budget, and splits TOTAL_CPU_DRAM_GB GB of eager # offload across the ranks; c160 and c192 shrink the per-rank KV cache. -override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, max-num-batched-tokens: 16384, kv-cache-memory-bytes: 53687091200, max-num-seqs: 1, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,100,200,300,400,500]}'}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, max-num-batched-tokens: 16384, kv-cache-memory-bytes: 53687091200, max-num-seqs: 4, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,100,200,300,400,500]}'}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} -override_tp8_c6: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, max-num-batched-tokens: 16384, kv-cache-memory-bytes: 53687091200, max-num-seqs: 6, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,100,200,300,400,500]}'}}}, benchmark: {env: {CONC: '6', KV_OFFLOADING: none}}} -override_tp8_c10: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, max-num-batched-tokens: 16384, kv-cache-memory-bytes: 53687091200, max-num-seqs: 10, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,100,200,300,400,500]}'}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: none}}} -override_tp8_c14: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, max-num-batched-tokens: 16384, kv-cache-memory-bytes: 53687091200, max-num-seqs: 14, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,100,200,300,400,500]}'}}}, benchmark: {env: {CONC: '14', KV_OFFLOADING: none}}} -override_tp8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, max-num-batched-tokens: 16384, kv-cache-memory-bytes: 53687091200, max-num-seqs: 16, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,100,105,112,200,300,400,500]}'}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: none}}} -override_dep8_c32: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 16, long-prefill-token-threshold: 512, kv-cache-memory-bytes: 37580963840, max-num-batched-tokens: 8248, max-num-seqs: 8, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1947', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} -override_dep8_c64: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 16, long-prefill-token-threshold: 512, kv-cache-memory-bytes: 37580963840, max-num-batched-tokens: 8304, max-num-seqs: 16, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1947', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} -override_dep8_c96: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 16, long-prefill-token-threshold: 512, kv-cache-memory-bytes: 37580963840, max-num-batched-tokens: 8360, max-num-seqs: 24, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '96', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1947', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} -override_dep8_c128: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 16, long-prefill-token-threshold: 512, kv-cache-memory-bytes: 37580963840, max-num-batched-tokens: 8416, max-num-seqs: 32, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '128', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1947', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} -override_dep8_c160: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 16, long-prefill-token-threshold: 512, kv-cache-memory-bytes: 36507222016, max-num-batched-tokens: 8472, max-num-seqs: 40, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224,231,238,245,252,259,266,273,280],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '160', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1947', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} -override_dep8_c192: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 16, long-prefill-token-threshold: 512, kv-cache-memory-bytes: 36507222016, max-num-batched-tokens: 8528, max-num-seqs: 48, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224,231,238,245,252,259,266,273,280,287,294,301,308,315,322,329,336],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '192', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1947', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + max-num-batched-tokens: 16384 + kv-cache-memory-bytes: 53687091200 + max-num-seqs: 1 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,100,200,300,400,500]}' + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + max-num-batched-tokens: 16384 + kv-cache-memory-bytes: 53687091200 + max-num-seqs: 4 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,100,200,300,400,500]}' + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c6: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + max-num-batched-tokens: 16384 + kv-cache-memory-bytes: 53687091200 + max-num-seqs: 6 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,100,200,300,400,500]}' + benchmark: + env: + CONC: '6' + KV_OFFLOADING: none + +override_tp8_c10: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + max-num-batched-tokens: 16384 + kv-cache-memory-bytes: 53687091200 + max-num-seqs: 10 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,100,200,300,400,500]}' + benchmark: + env: + CONC: '10' + KV_OFFLOADING: none + +override_tp8_c14: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + max-num-batched-tokens: 16384 + kv-cache-memory-bytes: 53687091200 + max-num-seqs: 14 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,100,200,300,400,500]}' + benchmark: + env: + CONC: '14' + KV_OFFLOADING: none + +override_tp8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + max-num-batched-tokens: 16384 + kv-cache-memory-bytes: 53687091200 + max-num-seqs: 16 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,100,105,112,200,300,400,500]}' + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_dep8_c32: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 16 + long-prefill-token-threshold: 512 + kv-cache-memory-bytes: 37580963840 + max-num-batched-tokens: 8248 + max-num-seqs: 8 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1947' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c64: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 16 + long-prefill-token-threshold: 512 + kv-cache-memory-bytes: 37580963840 + max-num-batched-tokens: 8304 + max-num-seqs: 16 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '64' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1947' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c96: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 16 + long-prefill-token-threshold: 512 + kv-cache-memory-bytes: 37580963840 + max-num-batched-tokens: 8360 + max-num-seqs: 24 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '96' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1947' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c128: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 16 + long-prefill-token-threshold: 512 + kv-cache-memory-bytes: 37580963840 + max-num-batched-tokens: 8416 + max-num-seqs: 32 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '128' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1947' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c160: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 16 + long-prefill-token-threshold: 512 + kv-cache-memory-bytes: 36507222016 + max-num-batched-tokens: 8472 + max-num-seqs: 40 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224,231,238,245,252,259,266,273,280],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '160' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1947' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c192: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 16 + long-prefill-token-threshold: 512 + kv-cache-memory-bytes: 36507222016 + max-num-batched-tokens: 8528 + max-num-seqs: 48 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224,231,238,245,252,259,266,273,280,287,294,301,308,315,322,329,336],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '192' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1947' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml index b3476ad511..800c798e7b 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml @@ -70,18 +70,409 @@ base: # offload so block hashes agree across ranks. DEP8 takes a larger prefill # budget and more memory headroom. DRAM points split TOTAL_CPU_DRAM_GB GB # across the ranks. -override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 2, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8],"mode":0}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto'}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 8, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32],"mode":0}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto'}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 2, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8],"mode":0}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto'}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 4, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16],"mode":0}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto'}}}, benchmark: {env: {CONC: '2', KV_OFFLOADING: none}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 8, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32],"mode":0}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto'}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} -override_tp4_c6: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 12, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48],"mode":0}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto'}}}, benchmark: {env: {CONC: '6', KV_OFFLOADING: none}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 16, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto'}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, data-parallel-size: 1, disable-custom-all-reduce: true, gpu-memory-utilization: 0.95, max-num-seqs: 32, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356000000000,"enable_cross_layers_blocks":"true","lazy_offload":true}}'}, env: {VLLM_ALLREDUCE_USE_FLASHINFER: '1', VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1424'}}} -override_dep4_c48: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 1, data-parallel-size: 4, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 8, long-prefill-token-threshold: 512, max-num-batched-tokens: 8192, gpu-memory-utilization: 0.95, max-num-seqs: 24, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356000000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '48', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1424', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} -override_dep4_c64: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 1, data-parallel-size: 4, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 8, long-prefill-token-threshold: 512, max-num-batched-tokens: 8192, gpu-memory-utilization: 0.95, max-num-seqs: 32, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356000000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1424', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} -override_dep8_c128: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 8, long-prefill-token-threshold: 512, max-num-batched-tokens: 16384, gpu-memory-utilization: 0.92, max-num-seqs: 32, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '128', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '2849', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} -override_dep8_c256: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 8, long-prefill-token-threshold: 512, max-num-batched-tokens: 16384, gpu-memory-utilization: 0.92, max-num-seqs: 64, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '256', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '2849', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} -override_dep8_c384: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 8, long-prefill-token-threshold: 512, max-num-batched-tokens: 16384, gpu-memory-utilization: 0.92, max-num-seqs: 96, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,292,296,300,304,308,312,316,320,324,328,332,336,340,344,348,352,356,360,364,368,372,376,380,384],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '384', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '2849', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} -override_dep8_c512: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 8, long-prefill-token-threshold: 512, max-num-batched-tokens: 16384, gpu-memory-utilization: 0.92, max-num-seqs: 128, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,292,296,300,304,308,312,316,320,324,328,332,336,340,344,348,352,356,360,364,368,372,376,380,384,388,392,396,400,404,408,412,416,420,424,428,432,436,440,444,448,452,456,460,464,468,472,476,480,484,488,492,496,500,504,508,512],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '512', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '2849', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} -override_dep8_c576: {frontend: {type: vllm-router, args: {policy: consistent_hash, prometheus-host: 127.0.0.1, prometheus-port: 18000, request-timeout-secs: 14400, disable-retries: true}}, setup_script: vllm-router-0.1.14.sh, roles: {agg: {gpus: 8, args: {tensor-parallel-size: 1, data-parallel-size: 8, enable-expert-parallel: true, enable-ep-weight-filter: true, moe-backend: deep_gemm_amxf4_mega_moe, prefill-schedule-interval: 8, long-prefill-token-threshold: 512, max-num-batched-tokens: 16384, gpu-memory-utilization: 0.92, max-num-seqs: 144, compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,292,296,300,304,308,312,316,320,324,328,332,336,340,344,348,352,356,360,364,368,372,376,380,384,388,392,396,400,404,408,412,416,420,424,428,432,436,440,444,448,452,456,460,464,468,472,476,480,484,488,492,496,500,504,508,512,516,520,524,528,532,536,540,544,548,552,556,560,564,568,572,576],"mode":0}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'}, env: {PYTORCH_ALLOC_CONF: 'expandable_segments:True', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '576', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '2849', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 2 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8],"mode":0}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 8 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32],"mode":0}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 2 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8],"mode":0}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 4 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16],"mode":0}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 8 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32],"mode":0}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp4_c6: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 12 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48],"mode":0}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + benchmark: + env: + CONC: '6' + KV_OFFLOADING: none + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 16 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 32 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356000000000,"enable_cross_layers_blocks":"true","lazy_offload":true}}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '16' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1424' + +override_dep4_c48: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 1 + data-parallel-size: 4 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 512 + max-num-batched-tokens: 8192 + gpu-memory-utilization: 0.95 + max-num-seqs: 24 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356000000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '48' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1424' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep4_c64: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 1 + data-parallel-size: 4 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 512 + max-num-batched-tokens: 8192 + gpu-memory-utilization: 0.95 + max-num-seqs: 32 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356000000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '64' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1424' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c128: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 512 + max-num-batched-tokens: 16384 + gpu-memory-utilization: 0.92 + max-num-seqs: 32 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '128' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2849' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c256: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 512 + max-num-batched-tokens: 16384 + gpu-memory-utilization: 0.92 + max-num-seqs: 64 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '256' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2849' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c384: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 512 + max-num-batched-tokens: 16384 + gpu-memory-utilization: 0.92 + max-num-seqs: 96 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,292,296,300,304,308,312,316,320,324,328,332,336,340,344,348,352,356,360,364,368,372,376,380,384],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '384' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2849' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c512: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 512 + max-num-batched-tokens: 16384 + gpu-memory-utilization: 0.92 + max-num-seqs: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,292,296,300,304,308,312,316,320,324,328,332,336,340,344,348,352,356,360,364,368,372,376,380,384,388,392,396,400,404,408,412,416,420,424,428,432,436,440,444,448,452,456,460,464,468,472,476,480,484,488,492,496,500,504,508,512],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '512' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2849' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c576: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 512 + max-num-batched-tokens: 16384 + gpu-memory-utilization: 0.92 + max-num-seqs: 144 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,292,296,300,304,308,312,316,320,324,328,332,336,340,344,348,352,356,360,364,368,372,376,380,384,388,392,396,400,404,408,412,416,420,424,428,432,436,440,444,448,452,456,460,464,468,472,476,480,484,488,492,496,500,504,508,512,516,520,524,528,532,536,540,544,548,552,556,560,564,568,572,576],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '576' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2849' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml index 60220c4a3d..47d2f7616e 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml @@ -83,15 +83,149 @@ base: # interleaves prefill every 8 steps with a 16k long-prefill threshold, and # routes each conversation to one rank: the Router hashes X-Session-ID, which # AIPerf aliases from its stable X-Correlation-ID. -override_tp8_c1: {roles: {agg: {args: {max-num-seqs: 2}}}, benchmark: {env: {CONC: '1'}}} -override_tp8_c4: {roles: {agg: {args: {max-num-seqs: 8}}}, benchmark: {env: {CONC: '4'}}} -override_tp8_c8: {roles: {agg: {args: {max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} -override_tp8_c16: {roles: {agg: {args: {max-num-seqs: 32}}}, benchmark: {env: {CONC: '16'}}} -override_tp8_c32: {roles: {agg: {args: {max-num-seqs: 64}}}, benchmark: {env: {CONC: '32'}}} -override_tp8_c40: {roles: {agg: {args: {max-num-seqs: 80}}}, benchmark: {env: {CONC: '40'}}} -override_tp8_c48: {roles: {agg: {args: {max-num-seqs: 96}}}, benchmark: {env: {CONC: '48'}}} -override_tp8_c56: {roles: {agg: {args: {max-num-seqs: 112}}}, benchmark: {env: {CONC: '56'}}} -override_tp8_c64: {roles: {agg: {args: {max-num-seqs: 128}}}, benchmark: {env: {CONC: '64'}}} -override_dep8_c64: {frontend: {type: vllm-router, args: {policy: consistent_hash, request-timeout-secs: 14400, disable-retries: true}, env: {SETUP_PIP_PACKAGES: 'vllm-router==0.1.14'}}, roles: {agg: {args: {tensor-parallel-size: 1, data-parallel-size: 8, prefill-schedule-interval: 8, long-prefill-token-threshold: 16384, max-num-seqs: 64}}}, benchmark: {env: {CONC: '64', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} -override_dep8_c128: {frontend: {type: vllm-router, args: {policy: consistent_hash, request-timeout-secs: 14400, disable-retries: true}, env: {SETUP_PIP_PACKAGES: 'vllm-router==0.1.14'}}, roles: {agg: {args: {tensor-parallel-size: 1, data-parallel-size: 8, prefill-schedule-interval: 8, long-prefill-token-threshold: 16384, max-num-seqs: 128}}}, benchmark: {env: {CONC: '128', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} -override_dep8_c192: {frontend: {type: vllm-router, args: {policy: consistent_hash, request-timeout-secs: 14400, disable-retries: true}, env: {SETUP_PIP_PACKAGES: 'vllm-router==0.1.14'}}, roles: {agg: {args: {tensor-parallel-size: 1, data-parallel-size: 8, prefill-schedule-interval: 8, long-prefill-token-threshold: 16384, max-num-seqs: 192}}}, benchmark: {env: {CONC: '192', AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'}}} +override_tp8_c1: + roles: + agg: + args: + max-num-seqs: 2 + benchmark: + env: + CONC: '1' + +override_tp8_c4: + roles: + agg: + args: + max-num-seqs: 8 + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + args: + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp8_c16: + roles: + agg: + args: + max-num-seqs: 32 + benchmark: + env: + CONC: '16' + +override_tp8_c32: + roles: + agg: + args: + max-num-seqs: 64 + benchmark: + env: + CONC: '32' + +override_tp8_c40: + roles: + agg: + args: + max-num-seqs: 80 + benchmark: + env: + CONC: '40' + +override_tp8_c48: + roles: + agg: + args: + max-num-seqs: 96 + benchmark: + env: + CONC: '48' + +override_tp8_c56: + roles: + agg: + args: + max-num-seqs: 112 + benchmark: + env: + CONC: '56' + +override_tp8_c64: + roles: + agg: + args: + max-num-seqs: 128 + benchmark: + env: + CONC: '64' + +override_dep8_c64: + frontend: + type: vllm-router + args: + policy: consistent_hash + request-timeout-secs: 14400 + disable-retries: true + env: + SETUP_PIP_PACKAGES: 'vllm-router==0.1.14' + roles: + agg: + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + prefill-schedule-interval: 8 + long-prefill-token-threshold: 16384 + max-num-seqs: 64 + benchmark: + env: + CONC: '64' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c128: + frontend: + type: vllm-router + args: + policy: consistent_hash + request-timeout-secs: 14400 + disable-retries: true + env: + SETUP_PIP_PACKAGES: 'vllm-router==0.1.14' + roles: + agg: + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + prefill-schedule-interval: 8 + long-prefill-token-threshold: 16384 + max-num-seqs: 128 + benchmark: + env: + CONC: '128' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c192: + frontend: + type: vllm-router + args: + policy: consistent_hash + request-timeout-secs: 14400 + disable-retries: true + env: + SETUP_PIP_PACKAGES: 'vllm-router==0.1.14' + roles: + agg: + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + prefill-schedule-interval: 8 + long-prefill-token-threshold: 16384 + max-num-seqs: 192 + benchmark: + env: + CONC: '192' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml index 2eb00ff3f9..c87f4758e3 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml @@ -62,15 +62,202 @@ base: # needs a larger static fraction and a smaller prefill chunk to fit, and # expandable segments against fragmentation. Saturation points get a longer # warmup drain. -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 128, max-running-requests: 2}, }}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 128, max-running-requests: 4}, }}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 256, max-running-requests: 8}, }}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 512, max-running-requests: 16}, }}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 1024, max-running-requests: 32}, }}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 2048, max-running-requests: 64}, }}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 4096, max-running-requests: 64}, }}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, chunked-prefill-size: 4096, swa-prefix-tails: 4096, max-running-requests: 64}, }}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 128, max-running-requests: 2}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}, }}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 256, max-running-requests: 4}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}, }}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 512, max-running-requests: 8}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}, }}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 1024, max-running-requests: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}, }}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 128 + max-running-requests: 2 + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 128 + max-running-requests: 4 + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 256 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 512 + max-running-requests: 16 + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 1024 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 2048 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 4096 + max-running-requests: 64 + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 4096 + max-running-requests: 64 + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 128 + max-running-requests: 2 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 256 + max-running-requests: 4 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 512 + max-running-requests: 8 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 1024 + max-running-requests: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml index 7415460727..61192ac5ea 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml @@ -64,19 +64,245 @@ base: # prefix tails are 64x CONC within [128, 4096]. TP2 raises the static fraction # (0.80 from c16, 0.85 from c32) and interleaves decode more often from c32. # Saturation points get a longer warmup drain. -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 128, max-running-requests: 2}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 128, max-running-requests: 4}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 256, max-running-requests: 8}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 512, max-running-requests: 16}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 1024, max-running-requests: 32}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 2048, max-running-requests: 64}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 4096, max-running-requests: 64}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.7, swa-prefix-tails: 4096, max-running-requests: 64}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.7, swa-prefix-tails: 128, max-running-requests: 2}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.7, swa-prefix-tails: 128, max-running-requests: 4}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.7, swa-prefix-tails: 256, max-running-requests: 8}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.7, swa-prefix-tails: 512, max-running-requests: 16}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.8, swa-prefix-tails: 1024, max-running-requests: 32}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.85, swa-prefix-tails: 2048, max-running-requests: 64, prefill-decode-interval: 4}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.85, swa-prefix-tails: 4096, max-running-requests: 64, prefill-decode-interval: 4}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.85, swa-prefix-tails: 4096, max-running-requests: 64, prefill-decode-interval: 4}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 128 + max-running-requests: 2 + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 128 + max-running-requests: 4 + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 256 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 512 + max-running-requests: 16 + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 1024 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 2048 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 4096 + max-running-requests: 64 + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 4096 + max-running-requests: 64 + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.7 + swa-prefix-tails: 128 + max-running-requests: 2 + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.7 + swa-prefix-tails: 128 + max-running-requests: 4 + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.7 + swa-prefix-tails: 256 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.7 + swa-prefix-tails: 512 + max-running-requests: 16 + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.8 + swa-prefix-tails: 1024 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.85 + swa-prefix-tails: 2048 + max-running-requests: 64 + prefill-decode-interval: 4 + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.85 + swa-prefix-tails: 4096 + max-running-requests: 64 + prefill-decode-interval: 4 + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.85 + swa-prefix-tails: 4096 + max-running-requests: 64 + prefill-decode-interval: 4 + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml index 7d3a2e810a..560eb34bee 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml @@ -63,19 +63,283 @@ base: # batch and expandable segments to fit. Admission is 2x CONC, capped at the graph # batch. SWA prefix tails are 64x CONC (TP2 128x), capped at 1024. Decode # interleaving applies to TP2 and c16. Saturation points get a longer warmup drain. -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 64, cuda-graph-max-bs-decode: 64, max-running-requests: 2}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 128, cuda-graph-max-bs-decode: 64, max-running-requests: 4}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 256, cuda-graph-max-bs-decode: 64, max-running-requests: 8}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 512, cuda-graph-max-bs-decode: 64, max-running-requests: 16}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 64, max-running-requests: 32, prefill-decode-interval: 16}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 64, max-running-requests: 64}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 64, max-running-requests: 64}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, mem-fraction-static: 0.7, chunked-prefill-size: 4096, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 64, max-running-requests: 64}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 128, cuda-graph-max-bs-decode: 16, max-running-requests: 2, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 256, cuda-graph-max-bs-decode: 16, max-running-requests: 4, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 512, cuda-graph-max-bs-decode: 16, max-running-requests: 8, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 16, max-running-requests: 16, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 16, max-running-requests: 16, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 16, max-running-requests: 16, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 16, max-running-requests: 16, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, mem-fraction-static: 0.92, chunked-prefill-size: 2048, swa-prefix-tails: 1024, cuda-graph-max-bs-decode: 16, max-running-requests: 16, prefill-decode-interval: 16}, env: {PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 64 + cuda-graph-max-bs-decode: 64 + max-running-requests: 2 + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 128 + cuda-graph-max-bs-decode: 64 + max-running-requests: 4 + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 256 + cuda-graph-max-bs-decode: 64 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 512 + cuda-graph-max-bs-decode: 64 + max-running-requests: 16 + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 64 + max-running-requests: 32 + prefill-decode-interval: 16 + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 64 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 64 + max-running-requests: 64 + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 64 + max-running-requests: 64 + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 128 + cuda-graph-max-bs-decode: 16 + max-running-requests: 2 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 256 + cuda-graph-max-bs-decode: 16 + max-running-requests: 4 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 512 + cuda-graph-max-bs-decode: 16 + max-running-requests: 8 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 16 + max-running-requests: 16 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 16 + max-running-requests: 16 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 16 + max-running-requests: 16 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 16 + max-running-requests: 16 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 16 + max-running-requests: 16 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml index adb716d218..e002f52cf1 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml @@ -65,19 +65,280 @@ base: # prefix tails are 64x CONC up to 4096 from c2. TP2 keeps row-sharded Engram # tables in host DRAM (TP4 fits them in HBM) and doubles the prefill chunk from # c64. Saturation points get a longer warmup drain. -override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 4096, max-running-requests: 2}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 4096, swa-prefix-tails: 128, max-running-requests: 4}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 4096, swa-prefix-tails: 256, max-running-requests: 8}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 4096, swa-prefix-tails: 512, max-running-requests: 16}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 4096, swa-prefix-tails: 1024, max-running-requests: 32}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 4096, swa-prefix-tails: 2048, max-running-requests: 64}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 8192, swa-prefix-tails: 4096, max-running-requests: 64}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, chunked-prefill-size: 8192, swa-prefix-tails: 4096, max-running-requests: 64}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1', SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, max-running-requests: 2}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, swa-prefix-tails: 128, max-running-requests: 4}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, swa-prefix-tails: 256, max-running-requests: 8}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, swa-prefix-tails: 512, max-running-requests: 16}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, swa-prefix-tails: 1024, max-running-requests: 32}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, swa-prefix-tails: 2048, max-running-requests: 64}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, swa-prefix-tails: 4096, max-running-requests: 64}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, chunked-prefill-size: 4096, swa-prefix-tails: 4096, max-running-requests: 64}, env: {SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0'}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 4096 + max-running-requests: 2 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 4096 + swa-prefix-tails: 128 + max-running-requests: 4 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 4096 + swa-prefix-tails: 256 + max-running-requests: 8 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 4096 + swa-prefix-tails: 512 + max-running-requests: 16 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 4096 + swa-prefix-tails: 1024 + max-running-requests: 32 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 4096 + swa-prefix-tails: 2048 + max-running-requests: 64 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 8192 + swa-prefix-tails: 4096 + max-running-requests: 64 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 8192 + swa-prefix-tails: 4096 + max-running-requests: 64 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + max-running-requests: 2 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + swa-prefix-tails: 128 + max-running-requests: 4 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + swa-prefix-tails: 256 + max-running-requests: 8 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + swa-prefix-tails: 512 + max-running-requests: 16 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + swa-prefix-tails: 1024 + max-running-requests: 32 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + swa-prefix-tails: 2048 + max-running-requests: 64 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + swa-prefix-tails: 4096 + max-running-requests: 64 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + swa-prefix-tails: 4096 + max-running-requests: 64 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml index d551234505..6f4767f0ce 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml @@ -87,16 +87,171 @@ base: # with decode graphs clamped to 64-128. The TP SWA tail reserve is 32x CONC from # c4 (8x below); the DP pool is per rank, and 64 tails/rank keeps the C16 TP # aggregate 512-tail reserve. -override_tp8_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 8}}}, benchmark: {env: {CONC: '1'}}} -override_tp8_c2: {roles: {agg: {args: {max-running-requests: 4, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 16}}}, benchmark: {env: {CONC: '2'}}} -override_tp8_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 128}}}, benchmark: {env: {CONC: '4'}}} -override_tp8_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 256}}}, benchmark: {env: {CONC: '8'}}} -override_tp8_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 512}}}, benchmark: {env: {CONC: '16'}}} -override_tp8_c20: {roles: {agg: {args: {max-running-requests: 40, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 640}}}, benchmark: {env: {CONC: '20'}}} +override_tp8_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 8 + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + args: + max-running-requests: 4 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 16 + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 128 + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 256 + benchmark: + env: + CONC: '8' + +override_tp8_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 512 + benchmark: + env: + CONC: '16' + +override_tp8_c20: + roles: + agg: + args: + max-running-requests: 40 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 640 + benchmark: + env: + CONC: '20' + # DP8 attention: the shipped MoE DSpark worker requires attn_tp=1. SGLang divides # the engine-wide 4096-token chunk by DP (512 tokens/rank). The DP-aware router # hashes AIPerf's per-session correlation ID so prefixes stay on one rank. -override_dp8_c4: {frontend: {type: sglang-router, args: {policy: consistent_hashing, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, max-running-requests: 8, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 64}}}, benchmark: {env: {CONC: '4', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'}}} -override_dp8_c8: {frontend: {type: sglang-router, args: {policy: consistent_hashing, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, max-running-requests: 16, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 64}}}, benchmark: {env: {CONC: '8', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'}}} -override_dp8_c16: {frontend: {type: sglang-router, args: {policy: consistent_hashing, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, max-running-requests: 32, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 64}}}, benchmark: {env: {CONC: '16', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'}}} -override_dp8_c20: {frontend: {type: sglang-router, args: {policy: consistent_hashing, request-id-headers: x-correlation-id, dp-aware: true, connect-timeout-secs: 900, request-timeout-secs: 14400, disable-health-check: true, disable-retries: true}}, roles: {agg: {args: {data-parallel-size: 8, enable-dp-attention: true, enable-dp-lm-head: true, max-running-requests: 40, cuda-graph-max-bs-decode: 64, swa-prefix-tails: 64}}}, benchmark: {env: {CONC: '20', AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'}}} +override_dp8_c4: + frontend: + type: sglang-router + args: + policy: consistent_hashing + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 8 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 64 + benchmark: + env: + CONC: '4' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + +override_dp8_c8: + frontend: + type: sglang-router + args: + policy: consistent_hashing + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 16 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 64 + benchmark: + env: + CONC: '8' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + +override_dp8_c16: + frontend: + type: sglang-router + args: + policy: consistent_hashing + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 32 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 64 + benchmark: + env: + CONC: '16' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + +override_dp8_c20: + frontend: + type: sglang-router + args: + policy: consistent_hashing + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 40 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 64 + benchmark: + env: + CONC: '20' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml index eccbd16327..0b55113278 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml @@ -83,19 +83,226 @@ base: # One variant per point. Admission is 2x CONC, capped at the decode graph batch; # saturation points carry a larger in-flight working set than the 30-minute # warmup drain allows. -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 2}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 4}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 8}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 16}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 32}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 64}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 64}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 64}, env: {DSV41_BLOCK32_TP: '4'}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 2}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '1', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp8_c2: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 4}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '2', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 8}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '4', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp8_c8: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 16}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '8', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 32}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '16', AGENTIC_WARMUP_GRACE_PERIOD: '1800'}}} -override_tp8_c32: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 64}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp8_c64: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 64}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '64', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} -override_tp8_c128: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 64}, env: {DSV41_BLOCK32_TP: '8'}}}, benchmark: {env: {CONC: '128', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 2 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 4 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 8 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 16 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 32 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 64 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 64 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 64 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 2 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 4 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 8 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp8_c8: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 16 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 32 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp8_c32: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 64 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c64: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 64 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c128: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 64 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml index 0a1fdec554..775db80214 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml @@ -82,9 +82,33 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' # One variant per point; the saturation point gets a longer warmup drain. -override_tp4_c1: {benchmark: {env: {CONC: '1'}}} -override_tp4_c2: {benchmark: {env: {CONC: '2'}}} -override_tp4_c4: {benchmark: {env: {CONC: '4'}}} -override_tp4_c8: {benchmark: {env: {CONC: '8'}}} -override_tp4_c16: {benchmark: {env: {CONC: '16'}}} -override_tp4_c32: {benchmark: {env: {CONC: '32', AGENTIC_WARMUP_GRACE_PERIOD: '3600'}}} +override_tp4_c1: + benchmark: + env: + CONC: '1' + +override_tp4_c2: + benchmark: + env: + CONC: '2' + +override_tp4_c4: + benchmark: + env: + CONC: '4' + +override_tp4_c8: + benchmark: + env: + CONC: '8' + +override_tp4_c16: + benchmark: + env: + CONC: '16' + +override_tp4_c32: + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml index 9ed63491af..82730b3561 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml @@ -58,19 +58,194 @@ base: # rank, so it caps batched tokens at 4096 (the indexer's 1M-wide logits buffer), # bounds the scheduler at 2x CONC within [16, 256] (FlashInfer autotune fails # below 16) and stops capturing above 512 tokens. -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '1'}}} -override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '2'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '4'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '8'}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 128}}}, benchmark: {env: {CONC: '16'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 256}}}, benchmark: {env: {CONC: '32'}}} -override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '64'}}} -override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 1024}}}, benchmark: {env: {CONC: '128'}}} -override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '1'}}} -override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '2'}}} -override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '4'}}} -override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} -override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 128, max-num-batched-tokens: 4096, max-num-seqs: 32}}}, benchmark: {env: {CONC: '16'}}} -override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 256, max-num-batched-tokens: 4096, max-num-seqs: 64}}}, benchmark: {env: {CONC: '32'}}} -override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 128}}}, benchmark: {env: {CONC: '64'}}} -override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 256}}}, benchmark: {env: {CONC: '128'}}} +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 128 + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 256 + benchmark: + env: + CONC: '32' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '64' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 1024 + benchmark: + env: + CONC: '128' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '1' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '2' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '4' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 128 + max-num-batched-tokens: 4096 + max-num-seqs: 32 + benchmark: + env: + CONC: '16' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 256 + max-num-batched-tokens: 4096 + max-num-seqs: 64 + benchmark: + env: + CONC: '32' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 128 + benchmark: + env: + CONC: '64' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 256 + benchmark: + env: + CONC: '128' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml index a73dafe593..34c77fe338 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml @@ -59,18 +59,198 @@ base: # batched-token limit matches the largest captured graph: CONC <= 4 and TP2 # CONC 128 capture up to 2046 tokens (the latter at 0.97 memory utilization), # every other point up to 8190. -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}', max-cudagraph-capture-size: 2046, max-num-batched-tokens: 2048}}}, benchmark: {env: {CONC: '1'}}} -override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}', max-cudagraph-capture-size: 2046, max-num-batched-tokens: 2048}}}, benchmark: {env: {CONC: '2'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}', max-cudagraph-capture-size: 2046, max-num-batched-tokens: 2048}}}, benchmark: {env: {CONC: '4'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '8'}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '16'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '32'}}} -override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '64'}}} -override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '128'}}} -override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}', max-cudagraph-capture-size: 2046, max-num-batched-tokens: 2048}}}, benchmark: {env: {CONC: '2'}}} -override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}', max-cudagraph-capture-size: 2046, max-num-batched-tokens: 2048}}}, benchmark: {env: {CONC: '4'}}} -override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '8'}}} -override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '16'}}} -override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '32'}}} -override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}', max-cudagraph-capture-size: 8190, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '64'}}} -override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}', max-cudagraph-capture-size: 2046, max-num-batched-tokens: 2048, gpu-memory-utilization: 0.97}}}, benchmark: {env: {CONC: '128'}}} +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}' + max-cudagraph-capture-size: 2046 + max-num-batched-tokens: 2048 + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}' + max-cudagraph-capture-size: 2046 + max-num-batched-tokens: 2048 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}' + max-cudagraph-capture-size: 2046 + max-num-batched-tokens: 2048 + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '32' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '64' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '128' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}' + max-cudagraph-capture-size: 2046 + max-num-batched-tokens: 2048 + benchmark: + env: + CONC: '2' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}' + max-cudagraph-capture-size: 2046 + max-num-batched-tokens: 2048 + benchmark: + env: + CONC: '4' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '8' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '16' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '32' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '64' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}' + max-cudagraph-capture-size: 2046 + max-num-batched-tokens: 2048 + gpu-memory-utilization: 0.97 + benchmark: + env: + CONC: '128' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml index 469d8026c9..64e0e7cca0 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml @@ -58,19 +58,194 @@ base: # each 256 GiB GPU, so it caps batched tokens at 4096 (the indexer's 1M-wide # logits buffer), bounds the scheduler at 2x CONC within [16, 256] (FlashInfer # autotune fails below 16) and stops capturing above 512 tokens. -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '1'}}} -override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '2'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '4'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '8'}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 128}}}, benchmark: {env: {CONC: '16'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 256}}}, benchmark: {env: {CONC: '32'}}} -override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '64'}}} -override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 1024}}}, benchmark: {env: {CONC: '128'}}} -override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '1'}}} -override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '2'}}} -override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '4'}}} -override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} -override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 128, max-num-batched-tokens: 4096, max-num-seqs: 32}}}, benchmark: {env: {CONC: '16'}}} -override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 256, max-num-batched-tokens: 4096, max-num-seqs: 64}}}, benchmark: {env: {CONC: '32'}}} -override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 128}}}, benchmark: {env: {CONC: '64'}}} -override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 256}}}, benchmark: {env: {CONC: '128'}}} +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 128 + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 256 + benchmark: + env: + CONC: '32' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '64' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 1024 + benchmark: + env: + CONC: '128' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '1' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '2' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '4' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 128 + max-num-batched-tokens: 4096 + max-num-seqs: 32 + benchmark: + env: + CONC: '16' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 256 + max-num-batched-tokens: 4096 + max-num-seqs: 64 + benchmark: + env: + CONC: '32' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 128 + benchmark: + env: + CONC: '64' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 256 + benchmark: + env: + CONC: '128' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml index 806b8555d6..1b5e2787e2 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml @@ -58,19 +58,194 @@ base: # each 277 GiB GPU, so it caps batched tokens at 4096 (the indexer's 1M-wide # logits buffer), bounds the scheduler at 2x CONC within [16, 256] (FlashInfer # autotune fails below 16) and stops capturing above 512 tokens. -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '1'}}} -override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '2'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '4'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '8'}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 128}}}, benchmark: {env: {CONC: '16'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 256}}}, benchmark: {env: {CONC: '32'}}} -override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '64'}}} -override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 1024}}}, benchmark: {env: {CONC: '128'}}} -override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '1'}}} -override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '2'}}} -override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '4'}}} -override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} -override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 128, max-num-batched-tokens: 4096, max-num-seqs: 32}}}, benchmark: {env: {CONC: '16'}}} -override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 256, max-num-batched-tokens: 4096, max-num-seqs: 64}}}, benchmark: {env: {CONC: '32'}}} -override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 128}}}, benchmark: {env: {CONC: '64'}}} -override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 256}}}, benchmark: {env: {CONC: '128'}}} +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 128 + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 256 + benchmark: + env: + CONC: '32' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '64' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 1024 + benchmark: + env: + CONC: '128' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '1' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '2' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '4' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 128 + max-num-batched-tokens: 4096 + max-num-seqs: 32 + benchmark: + env: + CONC: '16' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 256 + max-num-batched-tokens: 4096 + max-num-seqs: 64 + benchmark: + env: + CONC: '32' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 128 + benchmark: + env: + CONC: '64' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 256 + benchmark: + env: + CONC: '128' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml index 63d067ec79..d9a4abce41 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml @@ -68,11 +68,98 @@ base: # One variant per point. The scheduler batch is 2x CONC (AgentX subagent # fan-out) instead of vLLM's 1024 default; graph capture is the smallest power of # two covering that batch x (1 + 5 drafts), up to 2048. -override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 2, max-cudagraph-capture-size: 16}}}, benchmark: {env: {CONC: '1'}}} -override_tp8_c2: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 4, max-cudagraph-capture-size: 32}}}, benchmark: {env: {CONC: '2'}}} -override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 8, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '4'}}} -override_tp8_c8: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 16, max-cudagraph-capture-size: 128}}}, benchmark: {env: {CONC: '8'}}} -override_tp8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 32, max-cudagraph-capture-size: 256}}}, benchmark: {env: {CONC: '16'}}} -override_tp8_c20: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 40, max-cudagraph-capture-size: 256}}}, benchmark: {env: {CONC: '20'}}} -override_tp8_c24: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 48, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '24'}}} -override_tp8_c28: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-num-seqs: 56, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '28'}}} +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 2 + max-cudagraph-capture-size: 16 + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 4 + max-cudagraph-capture-size: 32 + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 8 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 16 + max-cudagraph-capture-size: 128 + benchmark: + env: + CONC: '8' + +override_tp8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 32 + max-cudagraph-capture-size: 256 + benchmark: + env: + CONC: '16' + +override_tp8_c20: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 40 + max-cudagraph-capture-size: 256 + benchmark: + env: + CONC: '20' + +override_tp8_c24: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 48 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '24' + +override_tp8_c28: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 56 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '28' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml index cfbbeb835e..2818ab0c17 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml @@ -64,19 +64,194 @@ base: # batched tokens at 4096 (the indexer's 1M-wide logits buffer), bounds the # scheduler at 2x CONC within [16, 256] (FlashInfer autotune fails below 16) and # stops capturing above 512 tokens. -override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '1'}}} -override_tp8_c2: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '2'}}} -override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '4'}}} -override_tp8_c8: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 64}}}, benchmark: {env: {CONC: '8'}}} -override_tp8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 128}}}, benchmark: {env: {CONC: '16'}}} -override_tp8_c32: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 256}}}, benchmark: {env: {CONC: '32'}}} -override_tp8_c64: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '64'}}} -override_tp8_c128: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-cudagraph-capture-size: 1024}}}, benchmark: {env: {CONC: '128'}}} -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '1'}}} -override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '2'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '4'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 64, max-num-batched-tokens: 4096, max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 128, max-num-batched-tokens: 4096, max-num-seqs: 32}}}, benchmark: {env: {CONC: '16'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 256, max-num-batched-tokens: 4096, max-num-seqs: 64}}}, benchmark: {env: {CONC: '32'}}} -override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 128}}}, benchmark: {env: {CONC: '64'}}} -override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-cudagraph-capture-size: 512, max-num-batched-tokens: 4096, max-num-seqs: 256}}}, benchmark: {env: {CONC: '128'}}} +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '8' + +override_tp8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 128 + benchmark: + env: + CONC: '16' + +override_tp8_c32: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 256 + benchmark: + env: + CONC: '32' + +override_tp8_c64: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '64' + +override_tp8_c128: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 1024 + benchmark: + env: + CONC: '128' + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 128 + max-num-batched-tokens: 4096 + max-num-seqs: 32 + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 256 + max-num-batched-tokens: 4096 + max-num-seqs: 64 + benchmark: + env: + CONC: '32' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 128 + benchmark: + env: + CONC: '64' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 256 + benchmark: + env: + CONC: '128' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml index 7d649adfaa..f2f1956224 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml @@ -81,15 +81,134 @@ base: # One variant per point. There is no TP2 arm: a 192 GB card measured negative KV # memory at TP2 (run 35671005506). -override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '1'}}} -override_tp8_c2: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '2'}}} -override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '4'}}} -override_tp8_c8: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '8'}}} -override_tp8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '16'}}} -override_tp8_c32: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '32'}}} -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '1'}}} -override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '2'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '4'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '8'}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '16'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '32'}}} +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '8' + +override_tp8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '16' + +override_tp8_c32: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '32' + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '32' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml index 1d2c034422..ec9b792ede 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml @@ -82,21 +82,218 @@ base: # One variant per point. TP2 leaves ~145 GiB of offloaded weights per rank, so it # halves the indexer buffer, bounds the scheduler at 2x CONC within [16, 256] # and stops capturing above 512 tokens. -override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '1'}}} -override_tp8_c2: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '2'}}} -override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '4'}}} -override_tp8_c8: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '8'}}} -override_tp8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '16'}}} -override_tp8_c32: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, engram-config: '{"cpu_offload":false}'}}}, benchmark: {env: {CONC: '32'}}} -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '1'}}} -override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '2'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '4'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '8'}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '16'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}'}}}, benchmark: {env: {CONC: '32'}}} -override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-num-batched-tokens: 8192, max-num-seqs: 16, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '1'}}} -override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-num-batched-tokens: 8192, max-num-seqs: 16, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '2'}}} -override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-num-batched-tokens: 8192, max-num-seqs: 16, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '4'}}} -override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-num-batched-tokens: 8192, max-num-seqs: 16, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '8'}}} -override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-num-batched-tokens: 8192, max-num-seqs: 32, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '16'}}} -override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-num-batched-tokens: 8192, max-num-seqs: 64, max-cudagraph-capture-size: 512}}}, benchmark: {env: {CONC: '32'}}} +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '8' + +override_tp8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '16' + +override_tp8_c32: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '32' + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '32' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-num-batched-tokens: 8192 + max-num-seqs: 16 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '1' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-num-batched-tokens: 8192 + max-num-seqs: 16 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '2' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-num-batched-tokens: 8192 + max-num-seqs: 16 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '4' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-num-batched-tokens: 8192 + max-num-seqs: 16 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '8' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-num-batched-tokens: 8192 + max-num-seqs: 32 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '16' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-num-batched-tokens: 8192 + max-num-seqs: 64 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '32' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml index d6bf50a85d..9ab84be6dd 100644 --- a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml @@ -75,19 +75,211 @@ base: # 2x CONC (at least 128 sequences) x (1 + 5 drafts), up to 2048 tokens. The # prefill chunk shrinks at TP2 from c64 and at c128 to buy KV room, and where # it drops below 1024 x 6 slots the scheduler is capped at the captured shape. -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":false}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '1'}}} -override_tp4_c2: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":false}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '2'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":false}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '4'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":false}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '8'}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":false}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '16'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":false}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '32'}}} -override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":false}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '64'}}} -override_tp4_c128: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 2048, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '128'}}} -override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '1'}}} -override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '2'}}} -override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '4'}}} -override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '8'}}} -override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '16'}}} -override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 16384}}}, benchmark: {env: {CONC: '32'}}} -override_tp2_c64: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 1024, max-num-batched-tokens: 8192}}}, benchmark: {env: {CONC: '64'}}} -override_tp2_c128: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, engram-config: '{"cpu_offload":true}', max-cudagraph-capture-size: 2048, max-num-batched-tokens: 4096, max-num-seqs: 256}}}, benchmark: {env: {CONC: '128'}}} +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":false}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":false}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":false}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":false}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":false}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":false}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '32' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":false}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '64' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '128' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '1' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '2' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '4' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '8' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '16' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '32' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '64' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 4096 + max-num-seqs: 256 + benchmark: + env: + CONC: '128' diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml index 33ce6a593c..e5672aa6f3 100644 --- a/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml @@ -50,10 +50,86 @@ base: # One variant per point. Admission is 2x CONC with graphs captured up to it. The # draft is five tokens through concurrency 8 and four above it. -override_tp4_c2: {roles: {agg: {gpus: 4, args: {max-num-seqs: 4, cudagraph-capture-sizes: '[1,2,4]', num-speculative-tokens: 5}}}, benchmark: {env: {CONC: '2'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {max-num-seqs: 8, cudagraph-capture-sizes: '[1,2,4,8]', num-speculative-tokens: 5}}}, benchmark: {env: {CONC: '4'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {max-num-seqs: 16, cudagraph-capture-sizes: '[1,2,4,8,12,16]', num-speculative-tokens: 5}}}, benchmark: {env: {CONC: '8'}}} -override_tp4_c10: {roles: {agg: {gpus: 4, args: {max-num-seqs: 20, cudagraph-capture-sizes: '[1,2,4,8,12,16,20]', num-speculative-tokens: 4}}}, benchmark: {env: {CONC: '10'}}} -override_tp8_c1: {roles: {agg: {gpus: 8, args: {max-num-seqs: 2, cudagraph-capture-sizes: '[1,2]', num-speculative-tokens: 5}}}, benchmark: {env: {CONC: '1'}}} -override_tp8_c2: {roles: {agg: {gpus: 8, args: {max-num-seqs: 4, cudagraph-capture-sizes: '[1,2,4]', num-speculative-tokens: 5}}}, benchmark: {env: {CONC: '2'}}} -override_tp8_c4: {roles: {agg: {gpus: 8, args: {max-num-seqs: 8, cudagraph-capture-sizes: '[1,2,4,8]', num-speculative-tokens: 5}}}, benchmark: {env: {CONC: '4'}}} +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 4 + cudagraph-capture-sizes: '[1,2,4]' + num-speculative-tokens: 5 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 8 + cudagraph-capture-sizes: '[1,2,4,8]' + num-speculative-tokens: 5 + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 16 + cudagraph-capture-sizes: '[1,2,4,8,12,16]' + num-speculative-tokens: 5 + benchmark: + env: + CONC: '8' + +override_tp4_c10: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 20 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20]' + num-speculative-tokens: 4 + benchmark: + env: + CONC: '10' + +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + max-num-seqs: 2 + cudagraph-capture-sizes: '[1,2]' + num-speculative-tokens: 5 + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + max-num-seqs: 4 + cudagraph-capture-sizes: '[1,2,4]' + num-speculative-tokens: 5 + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + max-num-seqs: 8 + cudagraph-capture-sizes: '[1,2,4,8]' + num-speculative-tokens: 5 + benchmark: + env: + CONC: '4' diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml index 05f1568b50..8add44971e 100644 --- a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml @@ -72,8 +72,57 @@ base: # capped at 64. MLA ranks hold complete per-token KV, so ratio 0.75 gives too # few host slots past C8; C12/C16 pin a 169 GB/rank pool (~1,662 GB across TP8 # with the DSA indexer, within the 1,731 GB budget). -override_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 2, hicache-ratio: 0.75}}}, benchmark: {env: {CONC: '1'}}} -override_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 8, hicache-ratio: 0.75}}}, benchmark: {env: {CONC: '4'}}} -override_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 16, hicache-ratio: 0.75}}}, benchmark: {env: {CONC: '8'}}} -override_c12: {roles: {agg: {args: {max-running-requests: 24, cuda-graph-max-bs: 24, hicache-size: 169}}}, benchmark: {env: {CONC: '12'}}} -override_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 32, hicache-size: 169}}}, benchmark: {env: {CONC: '16'}}} +override_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 2 + hicache-ratio: 0.75 + benchmark: + env: + CONC: '1' + +override_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 8 + hicache-ratio: 0.75 + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 16 + hicache-ratio: 0.75 + benchmark: + env: + CONC: '8' + +override_c12: + roles: + agg: + args: + max-running-requests: 24 + cuda-graph-max-bs: 24 + hicache-size: 169 + benchmark: + env: + CONC: '12' + +override_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 32 + hicache-size: 169 + benchmark: + env: + CONC: '16' diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml index 88d37041c1..5aa940cc33 100644 --- a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml @@ -72,8 +72,57 @@ base: # capped at 64. MLA ranks hold complete per-token KV, so ratio 0.75 gives too # few host slots past C8; C12/C16 pin a 169 GB/rank pool (~1,662 GB across TP8 # with the DSA indexer, within the 1,731 GB budget). -override_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 2, hicache-ratio: 0.75}}}, benchmark: {env: {CONC: '1'}}} -override_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 8, hicache-ratio: 0.75}}}, benchmark: {env: {CONC: '4'}}} -override_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 16, hicache-ratio: 0.75}}}, benchmark: {env: {CONC: '8'}}} -override_c12: {roles: {agg: {args: {max-running-requests: 24, cuda-graph-max-bs: 24, hicache-size: 169}}}, benchmark: {env: {CONC: '12'}}} -override_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 32, hicache-size: 169}}}, benchmark: {env: {CONC: '16'}}} +override_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 2 + hicache-ratio: 0.75 + benchmark: + env: + CONC: '1' + +override_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 8 + hicache-ratio: 0.75 + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 16 + hicache-ratio: 0.75 + benchmark: + env: + CONC: '8' + +override_c12: + roles: + agg: + args: + max-running-requests: 24 + cuda-graph-max-bs: 24 + hicache-size: 169 + benchmark: + env: + CONC: '12' + +override_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 32 + hicache-size: 169 + benchmark: + env: + CONC: '16' diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml index 573ecd9fce..e32b72696a 100644 --- a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml @@ -70,14 +70,112 @@ base: # One variant per point: admission is 2x CONC and the graph batch matches it, # capped at 64. -override_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 2}}}, benchmark: {env: {CONC: '1'}}} -override_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '4'}}} -override_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '8'}}} -override_c12: {roles: {agg: {args: {max-running-requests: 24, cuda-graph-max-bs: 24}}}, benchmark: {env: {CONC: '12'}}} -override_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '16'}}} -override_c20: {roles: {agg: {args: {max-running-requests: 40, cuda-graph-max-bs: 40}}}, benchmark: {env: {CONC: '20'}}} -override_c24: {roles: {agg: {args: {max-running-requests: 48, cuda-graph-max-bs: 48}}}, benchmark: {env: {CONC: '24'}}} -override_c28: {roles: {agg: {args: {max-running-requests: 56, cuda-graph-max-bs: 56}}}, benchmark: {env: {CONC: '28'}}} -override_c32: {roles: {agg: {args: {max-running-requests: 64, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '32'}}} -override_c40: {roles: {agg: {args: {max-running-requests: 80, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '40'}}} -override_c48: {roles: {agg: {args: {max-running-requests: 96, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '48'}}} +override_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 2 + benchmark: + env: + CONC: '1' + +override_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '8' + +override_c12: + roles: + agg: + args: + max-running-requests: 24 + cuda-graph-max-bs: 24 + benchmark: + env: + CONC: '12' + +override_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '16' + +override_c20: + roles: + agg: + args: + max-running-requests: 40 + cuda-graph-max-bs: 40 + benchmark: + env: + CONC: '20' + +override_c24: + roles: + agg: + args: + max-running-requests: 48 + cuda-graph-max-bs: 48 + benchmark: + env: + CONC: '24' + +override_c28: + roles: + agg: + args: + max-running-requests: 56 + cuda-graph-max-bs: 56 + benchmark: + env: + CONC: '28' + +override_c32: + roles: + agg: + args: + max-running-requests: 64 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '32' + +override_c40: + roles: + agg: + args: + max-running-requests: 80 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '40' + +override_c48: + roles: + agg: + args: + max-running-requests: 96 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '48' diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml index b3b183dfab..cfcbf7d3f1 100644 --- a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml @@ -72,8 +72,52 @@ base: # One variant per point: admission is 2x CONC and the graph batch matches it, # capped at 64. -override_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 2}}}, benchmark: {env: {CONC: '1'}}} -override_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '4'}}} -override_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '8'}}} -override_c12: {roles: {agg: {args: {max-running-requests: 24, cuda-graph-max-bs: 24}}}, benchmark: {env: {CONC: '12'}}} -override_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '16'}}} +override_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 2 + benchmark: + env: + CONC: '1' + +override_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '8' + +override_c12: + roles: + agg: + args: + max-running-requests: 24 + cuda-graph-max-bs: 24 + benchmark: + env: + CONC: '12' + +override_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '16' diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml index 3af26916f3..f329216a2d 100644 --- a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml @@ -71,10 +71,72 @@ base: AIPERF_BENCHMARK_GRACE_PERIOD: '1800' # One variant per point. Admission and the decode graph batch are 2x CONC. -override_tp8_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 2}}}, benchmark: {env: {CONC: '1'}}} -override_tp8_c2: {roles: {agg: {args: {max-running-requests: 4, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '2'}}} -override_tp8_c3: {roles: {agg: {args: {max-running-requests: 6, cuda-graph-max-bs: 6}}}, benchmark: {env: {CONC: '3'}}} -override_tp8_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '4'}}} -override_tp8_c5: {roles: {agg: {args: {max-running-requests: 10, cuda-graph-max-bs: 10}}}, benchmark: {env: {CONC: '5'}}} -override_tp8_c6: {roles: {agg: {args: {max-running-requests: 12, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '6'}}} -override_tp8_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '8'}}} +override_tp8_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 2 + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + args: + max-running-requests: 4 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '2' + +override_tp8_c3: + roles: + agg: + args: + max-running-requests: 6 + cuda-graph-max-bs: 6 + benchmark: + env: + CONC: '3' + +override_tp8_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '4' + +override_tp8_c5: + roles: + agg: + args: + max-running-requests: 10 + cuda-graph-max-bs: 10 + benchmark: + env: + CONC: '5' + +override_tp8_c6: + roles: + agg: + args: + max-running-requests: 12 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '6' + +override_tp8_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '8' diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml index 6e7956ce60..d2469415cc 100644 --- a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml @@ -58,12 +58,146 @@ base: # One variant per point. Admission is 2x CONC; the decode graph batch follows it # up to 64. The TP4 HiCache tier is 180 GB per rank. -override_tp4_c4_hicache: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, max-running-requests: 8, cuda-graph-max-bs-decode: 8, enable-hierarchical-cache: true, hicache-size: 180, hicache-write-policy: write_through}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: dram}}} -override_tp4_c8_hicache: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, max-running-requests: 16, cuda-graph-max-bs-decode: 16, enable-hierarchical-cache: true, hicache-size: 180, hicache-write-policy: write_through}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: dram}}} -override_tp4_c10_hicache: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, max-running-requests: 20, cuda-graph-max-bs-decode: 20, enable-hierarchical-cache: true, hicache-size: 180, hicache-write-policy: write_through}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: dram}}} -override_tp4_c12_hicache: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, max-running-requests: 24, cuda-graph-max-bs-decode: 24, enable-hierarchical-cache: true, hicache-size: 180, hicache-write-policy: write_through}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: dram}}} -override_tp4_c14_hicache: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, max-running-requests: 28, cuda-graph-max-bs-decode: 28, enable-hierarchical-cache: true, hicache-size: 180, hicache-write-policy: write_through}}}, benchmark: {env: {CONC: '14', KV_OFFLOADING: dram}}} -override_tp4_c16_hicache: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, max-running-requests: 32, cuda-graph-max-bs-decode: 32, enable-hierarchical-cache: true, hicache-size: 180, hicache-write-policy: write_through}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: dram}}} -override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 2, cuda-graph-max-bs-decode: 2}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp8_c2: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 4, cuda-graph-max-bs-decode: 4}}}, benchmark: {env: {CONC: '2', KV_OFFLOADING: none}}} -override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 8, cuda-graph-max-bs-decode: 8}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} +override_tp4_c4_hicache: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_through + benchmark: + env: + CONC: '4' + KV_OFFLOADING: dram + +override_tp4_c8_hicache: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_through + benchmark: + env: + CONC: '8' + KV_OFFLOADING: dram + +override_tp4_c10_hicache: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + max-running-requests: 20 + cuda-graph-max-bs-decode: 20 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_through + benchmark: + env: + CONC: '10' + KV_OFFLOADING: dram + +override_tp4_c12_hicache: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + max-running-requests: 24 + cuda-graph-max-bs-decode: 24 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_through + benchmark: + env: + CONC: '12' + KV_OFFLOADING: dram + +override_tp4_c14_hicache: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + max-running-requests: 28 + cuda-graph-max-bs-decode: 28 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_through + benchmark: + env: + CONC: '14' + KV_OFFLOADING: dram + +override_tp4_c16_hicache: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_through + benchmark: + env: + CONC: '16' + KV_OFFLOADING: dram + +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 4 + cuda-graph-max-bs-decode: 4 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml index 0fc956f57d..28a4b3cc63 100644 --- a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml @@ -59,10 +59,115 @@ base: # One variant per point. Admission is 2x CONC; the decode graph batch follows it # up to 64. HiCache holds 1.5x the device KV pool and skips non-reusable blocks. -override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 2, cuda-graph-max-bs-decode: 2}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp8_c2: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 4, cuda-graph-max-bs-decode: 4}}}, benchmark: {env: {CONC: '2', KV_OFFLOADING: none}}} -override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 8, cuda-graph-max-bs-decode: 8}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} -override_tp8_c10: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 20, cuda-graph-max-bs-decode: 20}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: none}}} -override_tp8_c8_hicache: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 16, cuda-graph-max-bs-decode: 16, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through_selective, hicache-io-backend: direct, hicache-mem-layout: page_first_direct}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: dram}}} -override_tp8_c12_hicache: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 24, cuda-graph-max-bs-decode: 24, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through_selective, hicache-io-backend: direct, hicache-mem-layout: page_first_direct}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: dram}}} -override_tp8_c16_hicache: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, max-running-requests: 32, cuda-graph-max-bs-decode: 32, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through_selective, hicache-io-backend: direct, hicache-mem-layout: page_first_direct}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: dram}}} +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 4 + cuda-graph-max-bs-decode: 4 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c10: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 20 + cuda-graph-max-bs-decode: 20 + benchmark: + env: + CONC: '10' + KV_OFFLOADING: none + +override_tp8_c8_hicache: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through_selective + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + benchmark: + env: + CONC: '8' + KV_OFFLOADING: dram + +override_tp8_c12_hicache: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 24 + cuda-graph-max-bs-decode: 24 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through_selective + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + benchmark: + env: + CONC: '12' + KV_OFFLOADING: dram + +override_tp8_c16_hicache: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through_selective + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + benchmark: + env: + CONC: '16' + KV_OFFLOADING: dram diff --git a/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml index db711b975e..ac5021e444 100644 --- a/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml @@ -85,5 +85,20 @@ base: # One variant per point. Full graphs cover every verify batch up to 2x CONC # times the eight-token verify window. -override_tp8_c1: {roles: {agg: {args: {cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]'}}}, benchmark: {env: {CONC: '1'}}} -override_tp8_c4: {roles: {agg: {args: {cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]'}}}, benchmark: {env: {CONC: '4'}}} +override_tp8_c1: + roles: + agg: + args: + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' + benchmark: + env: + CONC: '1' + +override_tp8_c4: + roles: + agg: + args: + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]' + benchmark: + env: + CONC: '4' diff --git a/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml index e2892344b4..e8d5e7039b 100644 --- a/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml @@ -113,14 +113,134 @@ base: # larger powers of two to 8192. Throughput runs switch DSpark to synthetic # rejection at the golden acceptance length; points above CONC 16 do not draft # and keep the matrix's mtp label. -override_c1: {roles: {agg: {args: {max-num-seqs: 2, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,64,128,256,512,1024,2048,4096,8192]}', speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'}}}, benchmark: {env: {CONC: '1'}}} -override_c2: {roles: {agg: {args: {max-num-seqs: 4, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,64,128,256,512,1024,2048,4096,8192]}', speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'}}}, benchmark: {env: {CONC: '2'}}} -override_c4: {roles: {agg: {args: {max-num-seqs: 8, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,128,256,512,1024,2048,4096,8192]}', speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'}}}, benchmark: {env: {CONC: '4'}}} -override_c8: {roles: {agg: {args: {max-num-seqs: 16, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,256,512,1024,2048,4096,8192]}', speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'}}}, benchmark: {env: {CONC: '8'}}} -override_c16: {roles: {agg: {args: {max-num-seqs: 32, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,256,512,1024,2048,4096,8192]}', speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'}}}, benchmark: {env: {CONC: '16'}}} -override_c24: {roles: {agg: {args: {max-num-seqs: 48, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,64,128,256,512,1024,2048,4096,8192]}'}}}, benchmark: {env: {CONC: '24', SPEC_DECODING: mtp}}} -override_c32: {roles: {agg: {args: {max-num-seqs: 64, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,128,256,512,1024,2048,4096,8192]}'}}}, benchmark: {env: {CONC: '32', SPEC_DECODING: mtp}}} -override_c40: {roles: {agg: {args: {max-num-seqs: 80, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,128,256,512,1024,2048,4096,8192]}'}}}, benchmark: {env: {CONC: '40', SPEC_DECODING: mtp}}} -override_c48: {roles: {agg: {args: {max-num-seqs: 96, gpu-memory-utilization: 0.92, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,128,256,512,1024,2048,4096,8192]}'}}}, benchmark: {env: {CONC: '48', SPEC_DECODING: mtp}}} -override_c56: {roles: {agg: {args: {max-num-seqs: 112, gpu-memory-utilization: 0.9, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,128,256,512,1024,2048,4096,8192]}'}}}, benchmark: {env: {CONC: '56', SPEC_DECODING: mtp}}} -override_c70: {roles: {agg: {args: {max-num-seqs: 140, gpu-memory-utilization: 0.9, compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,256,512,1024,2048,4096,8192]}'}}}, benchmark: {env: {CONC: '70', SPEC_DECODING: mtp}}} +override_c1: + roles: + agg: + args: + max-num-seqs: 2 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,64,128,256,512,1024,2048,4096,8192]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '1' + +override_c2: + roles: + agg: + args: + max-num-seqs: 4 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,64,128,256,512,1024,2048,4096,8192]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '2' + +override_c4: + roles: + agg: + args: + max-num-seqs: 8 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,128,256,512,1024,2048,4096,8192]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-num-seqs: 16 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,256,512,1024,2048,4096,8192]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '8' + +override_c16: + roles: + agg: + args: + max-num-seqs: 32 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,256,512,1024,2048,4096,8192]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '16' + +override_c24: + roles: + agg: + args: + max-num-seqs: 48 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,64,128,256,512,1024,2048,4096,8192]}' + benchmark: + env: + CONC: '24' + SPEC_DECODING: mtp + +override_c32: + roles: + agg: + args: + max-num-seqs: 64 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,128,256,512,1024,2048,4096,8192]}' + benchmark: + env: + CONC: '32' + SPEC_DECODING: mtp + +override_c40: + roles: + agg: + args: + max-num-seqs: 80 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,128,256,512,1024,2048,4096,8192]}' + benchmark: + env: + CONC: '40' + SPEC_DECODING: mtp + +override_c48: + roles: + agg: + args: + max-num-seqs: 96 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,128,256,512,1024,2048,4096,8192]}' + benchmark: + env: + CONC: '48' + SPEC_DECODING: mtp + +override_c56: + roles: + agg: + args: + max-num-seqs: 112 + gpu-memory-utilization: 0.9 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,128,256,512,1024,2048,4096,8192]}' + benchmark: + env: + CONC: '56' + SPEC_DECODING: mtp + +override_c70: + roles: + agg: + args: + max-num-seqs: 140 + gpu-memory-utilization: 0.9 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,256,512,1024,2048,4096,8192]}' + benchmark: + env: + CONC: '70' + SPEC_DECODING: mtp diff --git a/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml index 10982888eb..0621caa8e2 100644 --- a/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml @@ -82,9 +82,104 @@ base: # FULL_AND_PIECEWISE graphs cost KV pool, so only c1 and c4 use them. The # offload points split the 1799 GB DRAM budget evenly across the eight ranks; # identical prefixes must hash to identical block keys on every rank. -override_tp8_c1: {roles: {agg: {args: {max-num-seqs: 2, max-num-batched-tokens: 16384, compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]}', speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'}, env: {VLLM_USE_BREAKABLE_CUDAGRAPH: '1'}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: 'none'}}} -override_tp8_c4: {roles: {agg: {args: {max-num-seqs: 8, max-num-batched-tokens: 8192, compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":48,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48]}', speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":5,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'}, env: {VLLM_USE_BREAKABLE_CUDAGRAPH: '1'}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: 'none'}}} -override_tp8_c8_simple: {roles: {agg: {args: {max-num-seqs: 16, max-num-batched-tokens: 8192, compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":80,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80]}', speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}'}, env: {VLLM_USE_BREAKABLE_CUDAGRAPH: '0', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '1799'}}} -override_tp8_c10_simple: {roles: {agg: {args: {max-num-seqs: 20, max-num-batched-tokens: 8192, compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":100,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100]}', speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}'}, env: {VLLM_USE_BREAKABLE_CUDAGRAPH: '0', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '1799'}}} -override_tp8_c12_simple: {roles: {agg: {args: {max-num-seqs: 24, max-num-batched-tokens: 8192, compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":96,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96]}', speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}'}, env: {VLLM_USE_BREAKABLE_CUDAGRAPH: '0', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '1799'}}} -override_tp8_c14_simple: {roles: {agg: {args: {max-num-seqs: 28, max-num-batched-tokens: 8192, compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":112,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112]}', speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}', kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}'}, env: {VLLM_USE_BREAKABLE_CUDAGRAPH: '0', PYTHONHASHSEED: '42'}}}, benchmark: {env: {CONC: '14', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '1799'}}} +override_tp8_c1: + roles: + agg: + args: + max-num-seqs: 2 + max-num-batched-tokens: 16384 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '1' + benchmark: + env: + CONC: '1' + KV_OFFLOADING: 'none' + +override_tp8_c4: + roles: + agg: + args: + max-num-seqs: 8 + max-num-batched-tokens: 8192 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":48,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":5,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '1' + benchmark: + env: + CONC: '4' + KV_OFFLOADING: 'none' + +override_tp8_c8_simple: + roles: + agg: + args: + max-num-seqs: 16 + max-num-batched-tokens: 8192 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":80,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '8' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + +override_tp8_c10_simple: + roles: + agg: + args: + max-num-seqs: 20 + max-num-batched-tokens: 8192 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":100,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '10' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + +override_tp8_c12_simple: + roles: + agg: + args: + max-num-seqs: 24 + max-num-batched-tokens: 8192 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":96,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '12' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + +override_tp8_c14_simple: + roles: + agg: + args: + max-num-seqs: 28 + max-num-batched-tokens: 8192 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":112,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '14' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml index 62951121c5..df6b8d5165 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml @@ -57,17 +57,152 @@ base: # One variant per point. Admission is 2x CONC. TP4 shards the indexer across # ranks at the measured points (15, 20, 24, 28, 32). -override_tp4_c1: {roles: {agg: {gpus: 4, args: {max-num-seqs: 2}}}, benchmark: {env: {CONC: '1'}}} -override_tp4_c2: {roles: {agg: {gpus: 4, args: {max-num-seqs: 4}}}, benchmark: {env: {CONC: '2'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {max-num-seqs: 8}}}, benchmark: {env: {CONC: '4'}}} -override_tp4_c5: {roles: {agg: {gpus: 4, args: {max-num-seqs: 10}}}, benchmark: {env: {CONC: '5'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} -override_tp4_c10: {roles: {agg: {gpus: 4, args: {max-num-seqs: 20}}}, benchmark: {env: {CONC: '10'}}} -override_tp4_c12: {roles: {agg: {gpus: 4, args: {max-num-seqs: 24}}}, benchmark: {env: {CONC: '12'}}} -override_tp4_c15: {roles: {agg: {gpus: 4, args: {max-num-seqs: 30}, env: {ATOM_M3_INDEXER_CP: '1'}}}, benchmark: {env: {CONC: '15'}}} -override_tp4_c20: {roles: {agg: {gpus: 4, args: {max-num-seqs: 40}, env: {ATOM_M3_INDEXER_CP: '1'}}}, benchmark: {env: {CONC: '20'}}} -override_tp4_c24: {roles: {agg: {gpus: 4, args: {max-num-seqs: 48}, env: {ATOM_M3_INDEXER_CP: '1'}}}, benchmark: {env: {CONC: '24'}}} -override_tp4_c28: {roles: {agg: {gpus: 4, args: {max-num-seqs: 56}, env: {ATOM_M3_INDEXER_CP: '1'}}}, benchmark: {env: {CONC: '28'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {max-num-seqs: 64}, env: {ATOM_M3_INDEXER_CP: '1'}}}, benchmark: {env: {CONC: '32'}}} -override_tp2_c1: {roles: {agg: {gpus: 2, args: {max-num-seqs: 2}}}, benchmark: {env: {CONC: '1'}}} -override_tp2_c2: {roles: {agg: {gpus: 2, args: {max-num-seqs: 4}}}, benchmark: {env: {CONC: '2'}}} +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 2 + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 4 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 8 + benchmark: + env: + CONC: '4' + +override_tp4_c5: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 10 + benchmark: + env: + CONC: '5' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp4_c10: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 20 + benchmark: + env: + CONC: '10' + +override_tp4_c12: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 24 + benchmark: + env: + CONC: '12' + +override_tp4_c15: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 30 + env: + ATOM_M3_INDEXER_CP: '1' + benchmark: + env: + CONC: '15' + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 40 + env: + ATOM_M3_INDEXER_CP: '1' + benchmark: + env: + CONC: '20' + +override_tp4_c24: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 48 + env: + ATOM_M3_INDEXER_CP: '1' + benchmark: + env: + CONC: '24' + +override_tp4_c28: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 56 + env: + ATOM_M3_INDEXER_CP: '1' + benchmark: + env: + CONC: '28' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 64 + env: + ATOM_M3_INDEXER_CP: '1' + benchmark: + env: + CONC: '32' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + max-num-seqs: 2 + benchmark: + env: + CONC: '1' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + max-num-seqs: 4 + benchmark: + env: + CONC: '2' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml index 988a828038..870350371e 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml @@ -102,14 +102,331 @@ base: # One variant per point. The batch cap is CONC; graphs capture every batch up # to 20 and a thinned ladder above. The host pool is 250 GiB at TP4 and # 200 GiB at TP8. -override_tp4_c5: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 5, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '5', TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_c10: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 10, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '10', TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_c15: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 15, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '15', TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 20, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '20', TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_c25: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 25, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 19, 21, 23, 25]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '25', TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_c30: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 30, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 18, 20, 22, 24, 27, 30]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '30', TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_c35: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 35, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 17, 20, 23, 26, 29, 32, 35]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '35', TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_c40: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 40, cuda_graph_config: {batch_sizes: [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '40', TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_c45: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 45, cuda_graph_config: {batch_sizes: [2, 4, 6, 8, 10, 12, 14, 16, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45]}, kv_cache_config: {host_cache_size: 268435456000}}}}, benchmark: {env: {CONC: '45', TOTAL_CPU_DRAM_GB: '865'}}} -override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor_parallel_size: 8, max_batch_size: 1, cuda_graph_config: {batch_sizes: [1]}, kv_cache_config: {host_cache_size: 214748364800}}}}, benchmark: {env: {CONC: '1', TOTAL_CPU_DRAM_GB: '1731'}}} -override_tp8_c5: {roles: {agg: {gpus: 8, args: {tensor_parallel_size: 8, max_batch_size: 5, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5]}, kv_cache_config: {host_cache_size: 214748364800}}}}, benchmark: {env: {CONC: '5', TOTAL_CPU_DRAM_GB: '1731'}}} +override_tp4_c5: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 5 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '5' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c10: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 10 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '10' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c15: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 15 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '15' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 20 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '20' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c25: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 25 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 17 + - 19 + - 21 + - 23 + - 25 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '25' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c30: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 30 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 27 + - 30 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '30' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c35: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 35 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 14 + - 17 + - 20 + - 23 + - 26 + - 29 + - 32 + - 35 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '35' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c40: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 40 + cuda_graph_config: + batch_sizes: + - 2 + - 4 + - 6 + - 8 + - 10 + - 12 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 26 + - 28 + - 30 + - 32 + - 34 + - 36 + - 38 + - 40 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '40' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c45: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 45 + cuda_graph_config: + batch_sizes: + - 2 + - 4 + - 6 + - 8 + - 10 + - 12 + - 14 + - 16 + - 18 + - 21 + - 24 + - 27 + - 30 + - 33 + - 36 + - 39 + - 42 + - 45 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '45' + TOTAL_CPU_DRAM_GB: '865' + +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor_parallel_size: 8 + max_batch_size: 1 + cuda_graph_config: + batch_sizes: + - 1 + kv_cache_config: + host_cache_size: 214748364800 + benchmark: + env: + CONC: '1' + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_c5: + roles: + agg: + gpus: 8 + args: + tensor_parallel_size: 8 + max_batch_size: 5 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + kv_cache_config: + host_cache_size: 214748364800 + benchmark: + env: + CONC: '5' + TOTAL_CPU_DRAM_GB: '1731' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml index 6ede031d15..1e43fa70ae 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml @@ -100,14 +100,334 @@ base: # One variant per point. The batch limit is CONC with graphs captured at every # size to 20 and a thinned ladder at 25 and 30. The host pool is 288 GiB at TP8 # and about 362 GiB below. -override_tp2_c20: {roles: {agg: {gpus: 2, args: {tensor_parallel_size: 2, max_batch_size: 20, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '20', TOTAL_CPU_DRAM_GB: '599'}}} -override_tp2_c25: {roles: {agg: {gpus: 2, args: {tensor_parallel_size: 2, max_batch_size: 25, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 19, 21, 23, 25]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '25', TOTAL_CPU_DRAM_GB: '599'}}} -override_tp2_c30: {roles: {agg: {gpus: 2, args: {tensor_parallel_size: 2, max_batch_size: 30, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 18, 20, 22, 24, 27, 30]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '30', TOTAL_CPU_DRAM_GB: '599'}}} -override_tp4_c5: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 5, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '5', TOTAL_CPU_DRAM_GB: '1199'}}} -override_tp4_c10: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 10, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '10', TOTAL_CPU_DRAM_GB: '1199'}}} -override_tp4_c15: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 15, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '15', TOTAL_CPU_DRAM_GB: '1199'}}} -override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 20, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '20', TOTAL_CPU_DRAM_GB: '1199'}}} -override_tp4_c25: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 25, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 19, 21, 23, 25]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '25', TOTAL_CPU_DRAM_GB: '1199'}}} -override_tp4_c30: {roles: {agg: {gpus: 4, args: {tensor_parallel_size: 4, max_batch_size: 30, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 18, 20, 22, 24, 27, 30]}, kv_cache_config: {host_cache_size: 388554555392}}}}, benchmark: {env: {CONC: '30', TOTAL_CPU_DRAM_GB: '1199'}}} -override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor_parallel_size: 8, max_batch_size: 1, cuda_graph_config: {batch_sizes: [1]}, kv_cache_config: {host_cache_size: 309237645312}}}}, benchmark: {env: {CONC: '1', TOTAL_CPU_DRAM_GB: '2399'}}} -override_tp8_c5: {roles: {agg: {gpus: 8, args: {tensor_parallel_size: 8, max_batch_size: 5, cuda_graph_config: {batch_sizes: [1, 2, 3, 4, 5]}, kv_cache_config: {host_cache_size: 309237645312}}}}, benchmark: {env: {CONC: '5', TOTAL_CPU_DRAM_GB: '2399'}}} +override_tp2_c20: + roles: + agg: + gpus: 2 + args: + tensor_parallel_size: 2 + max_batch_size: 20 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '20' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c25: + roles: + agg: + gpus: 2 + args: + tensor_parallel_size: 2 + max_batch_size: 25 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 17 + - 19 + - 21 + - 23 + - 25 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '25' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c30: + roles: + agg: + gpus: 2 + args: + tensor_parallel_size: 2 + max_batch_size: 30 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 27 + - 30 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '30' + TOTAL_CPU_DRAM_GB: '599' + +override_tp4_c5: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 5 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '5' + TOTAL_CPU_DRAM_GB: '1199' + +override_tp4_c10: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 10 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '10' + TOTAL_CPU_DRAM_GB: '1199' + +override_tp4_c15: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 15 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '15' + TOTAL_CPU_DRAM_GB: '1199' + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 20 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '20' + TOTAL_CPU_DRAM_GB: '1199' + +override_tp4_c25: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 25 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 17 + - 19 + - 21 + - 23 + - 25 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '25' + TOTAL_CPU_DRAM_GB: '1199' + +override_tp4_c30: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 30 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 27 + - 30 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '30' + TOTAL_CPU_DRAM_GB: '1199' + +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor_parallel_size: 8 + max_batch_size: 1 + cuda_graph_config: + batch_sizes: + - 1 + kv_cache_config: + host_cache_size: 309237645312 + benchmark: + env: + CONC: '1' + TOTAL_CPU_DRAM_GB: '2399' + +override_tp8_c5: + roles: + agg: + gpus: 8 + args: + tensor_parallel_size: 8 + max_batch_size: 5 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + kv_cache_config: + host_cache_size: 309237645312 + benchmark: + env: + CONC: '5' + TOTAL_CPU_DRAM_GB: '2399' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml index 0c83164aba..9a12664c47 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml @@ -63,18 +63,212 @@ base: # One variant per point. DRAM points patch vLLM's SimpleCPUOffload worker for # heterogeneous layer regions and give it the whole host budget # (TOTAL_CPU_DRAM_GB GiB) with lazy offload. -override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp4_c5: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '5', KV_OFFLOADING: none}}} -override_tp4_c10: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: none}}} -override_tp4_c15: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '15', KV_OFFLOADING: none}}} -override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: none}}} -override_tp4_c15_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '15', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} -override_tp4_c20_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} -override_tp4_c25_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '25', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} -override_tp4_c30_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '30', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} -override_tp4_c32_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} -override_tp4_c34_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '34', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} -override_tp4_c36_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '36', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} -override_tp4_c38_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '38', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} -override_tp4_c40_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '40', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1082'}}} +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c5: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '5' + KV_OFFLOADING: none + +override_tp4_c10: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '10' + KV_OFFLOADING: none + +override_tp4_c15: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '15' + KV_OFFLOADING: none + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: none + +override_tp4_c15_dram: + setup_script: vllm-simple-kv-offload.sh + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '15' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c20_dram: + setup_script: vllm-simple-kv-offload.sh + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '20' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c25_dram: + setup_script: vllm-simple-kv-offload.sh + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '25' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c30_dram: + setup_script: vllm-simple-kv-offload.sh + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '30' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c32_dram: + setup_script: vllm-simple-kv-offload.sh + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c34_dram: + setup_script: vllm-simple-kv-offload.sh + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '34' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c36_dram: + setup_script: vllm-simple-kv-offload.sh + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '36' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c38_dram: + setup_script: vllm-simple-kv-offload.sh + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '38' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c40_dram: + setup_script: vllm-simple-kv-offload.sh + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '40' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml index d4bc31ace7..5913cada03 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml @@ -63,11 +63,100 @@ base: # One variant per point. DRAM points patch vLLM's SimpleCPUOffload worker for # heterogeneous layer regions and give it the whole host budget # (TOTAL_CPU_DRAM_GB GiB) with lazy offload. -override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp4_c5: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '5', KV_OFFLOADING: none}}} -override_tp4_c10: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: none}}} -override_tp4_c15: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '15', KV_OFFLOADING: none}}} -override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: none}}} -override_tp4_c30_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1099511627776,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '30', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1024'}}} -override_tp2_c24_dram: {setup_script: vllm-simple-kv-offload.sh, roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":804232626176,"lazy_offload":true}}'}, env: {VLLM_USE_SIMPLE_KV_OFFLOAD: '1'}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '749'}}} +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c5: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '5' + KV_OFFLOADING: none + +override_tp4_c10: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '10' + KV_OFFLOADING: none + +override_tp4_c15: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '15' + KV_OFFLOADING: none + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: none + +override_tp4_c30_dram: + setup_script: vllm-simple-kv-offload.sh + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1099511627776,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '30' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1024' + +override_tp2_c24_dram: + setup_script: vllm-simple-kv-offload.sh + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":804232626176,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '24' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '749' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml index 2f8f0f85ab..f30dcd6a43 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml @@ -65,11 +65,61 @@ base: # One variant per point. The scheduler batch is 2x CONC and graph capture covers # it at four tokens per sequence (three drafts). -override_tp8_c1: {roles: {agg: {args: {max-num-seqs: 2, max-cudagraph-capture-size: 8}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp8_c2: {roles: {agg: {args: {max-num-seqs: 4, max-cudagraph-capture-size: 16}}}, benchmark: {env: {CONC: '2', KV_OFFLOADING: none}}} -override_tp8_c3: {roles: {agg: {args: {max-num-seqs: 6, max-cudagraph-capture-size: 24}}}, benchmark: {env: {CONC: '3', KV_OFFLOADING: none}}} -override_tp8_c4: {roles: {agg: {args: {max-num-seqs: 8, max-cudagraph-capture-size: 32}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} -override_tp8_c5: {roles: {agg: {args: {max-num-seqs: 10, max-cudagraph-capture-size: 40}}}, benchmark: {env: {CONC: '5', KV_OFFLOADING: none}}} +override_tp8_c1: + roles: + agg: + args: + max-num-seqs: 2 + max-cudagraph-capture-size: 8 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c2: + roles: + agg: + args: + max-num-seqs: 4 + max-cudagraph-capture-size: 16 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + +override_tp8_c3: + roles: + agg: + args: + max-num-seqs: 6 + max-cudagraph-capture-size: 24 + benchmark: + env: + CONC: '3' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + args: + max-num-seqs: 8 + max-cudagraph-capture-size: 32 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c5: + roles: + agg: + args: + max-num-seqs: 10 + max-cudagraph-capture-size: 40 + benchmark: + env: + CONC: '5' + KV_OFFLOADING: none + # DRAM points offload KV to an embedded Mooncake store. Per rank: the host budget # (1731 GB = 1612 GiB) less the 414 GiB checkpoint page cache, over TP8, less the # 26 GiB CPU weight offload and the 4 GiB local buffer = 119 GB. @@ -80,12 +130,16 @@ override_tp8_c6_dram: - name: mooncake-master type: mooncake-master # The in-container install outlasts the default 120 s readiness window. - readiness: {port: 8700, timeout_seconds: 900} + readiness: + port: 8700 + timeout_seconds: 900 preamble: >- pip_install=(python3 -m pip install); python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages && pip_install+=(--break-system-packages); "${pip_install[@]}" --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 - args: ['--eviction_high_watermark_ratio=0.80', '--eviction_ratio=0.10'] + args: + - '--eviction_high_watermark_ratio=0.80' + - '--eviction_ratio=0.10' options: store_config: mode: embedded @@ -111,6 +165,7 @@ override_tp8_c6_dram: CONC: '6' KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '1731' + override_tp8_c8_dram: # The worker's Mooncake client and the master run the same pinned release. setup_script: vllm-mooncake-0.3.11.sh @@ -118,12 +173,16 @@ override_tp8_c8_dram: - name: mooncake-master type: mooncake-master # The in-container install outlasts the default 120 s readiness window. - readiness: {port: 8700, timeout_seconds: 900} + readiness: + port: 8700 + timeout_seconds: 900 preamble: >- pip_install=(python3 -m pip install); python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages && pip_install+=(--break-system-packages); "${pip_install[@]}" --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 - args: ['--eviction_high_watermark_ratio=0.80', '--eviction_ratio=0.10'] + args: + - '--eviction_high_watermark_ratio=0.80' + - '--eviction_ratio=0.10' options: store_config: mode: embedded diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml index 2d28db4595..36451e353f 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml @@ -64,12 +64,78 @@ base: # One variant per point. The scheduler batch is 2x CONC and graph capture covers # it at four tokens per sequence (three drafts). -override_tp8_c1: {roles: {agg: {args: {max-num-seqs: 2, max-cudagraph-capture-size: 8, gpu-memory-utilization: 0.95}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp8_c2: {roles: {agg: {args: {max-num-seqs: 4, max-cudagraph-capture-size: 16, gpu-memory-utilization: 0.95}}}, benchmark: {env: {CONC: '2', KV_OFFLOADING: none}}} -override_tp8_c4: {roles: {agg: {args: {max-num-seqs: 8, max-cudagraph-capture-size: 32, gpu-memory-utilization: 0.95}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} -override_tp8_c6: {roles: {agg: {args: {max-num-seqs: 12, max-cudagraph-capture-size: 48, gpu-memory-utilization: 0.95}}}, benchmark: {env: {CONC: '6', KV_OFFLOADING: none}}} -override_tp8_c8: {roles: {agg: {args: {max-num-seqs: 16, max-cudagraph-capture-size: 64, gpu-memory-utilization: 0.95}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} -override_tp8_c10: {roles: {agg: {args: {max-num-seqs: 20, max-cudagraph-capture-size: 80, gpu-memory-utilization: 0.95}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: none}}} +override_tp8_c1: + roles: + agg: + args: + max-num-seqs: 2 + max-cudagraph-capture-size: 8 + gpu-memory-utilization: 0.95 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c2: + roles: + agg: + args: + max-num-seqs: 4 + max-cudagraph-capture-size: 16 + gpu-memory-utilization: 0.95 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + args: + max-num-seqs: 8 + max-cudagraph-capture-size: 32 + gpu-memory-utilization: 0.95 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c6: + roles: + agg: + args: + max-num-seqs: 12 + max-cudagraph-capture-size: 48 + gpu-memory-utilization: 0.95 + benchmark: + env: + CONC: '6' + KV_OFFLOADING: none + +override_tp8_c8: + roles: + agg: + args: + max-num-seqs: 16 + max-cudagraph-capture-size: 64 + gpu-memory-utilization: 0.95 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp8_c10: + roles: + agg: + args: + max-num-seqs: 20 + max-cudagraph-capture-size: 80 + gpu-memory-utilization: 0.95 + benchmark: + env: + CONC: '10' + KV_OFFLOADING: none + # The DRAM point offloads KV to an embedded Mooncake store, leaving HBM headroom # at 0.90. Per rank: the host budget (1234 GB = 1149 GiB) less the 414 GiB # checkpoint page cache, over TP8, less the 4 GiB local buffer = 87 GB. @@ -80,12 +146,16 @@ override_tp8_c12_dram: - name: mooncake-master type: mooncake-master # The in-container install outlasts the default 120 s readiness window. - readiness: {port: 8700, timeout_seconds: 900} + readiness: + port: 8700 + timeout_seconds: 900 preamble: >- pip_install=(python3 -m pip install); python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages && pip_install+=(--break-system-packages); "${pip_install[@]}" --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 - args: ['--eviction_high_watermark_ratio=0.80', '--eviction_ratio=0.10'] + args: + - '--eviction_high_watermark_ratio=0.80' + - '--eviction_ratio=0.10' options: store_config: mode: embedded diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml index 21f2ade6b9..151ce47cc7 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml @@ -74,9 +74,67 @@ base: # One variant per point. Admission is 2x CONC. The DRAM point splits its # 1298 GB budget into one LMCache L1 shard per TP rank (1298 / 8 = 162 GB). -override_tp8_c2: {roles: {agg: {args: {max-num-seqs: 4}}}, benchmark: {env: {CONC: '2', KV_OFFLOADING: 'none'}}} -override_tp8_c4: {roles: {agg: {args: {max-num-seqs: 8}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: 'none'}}} -override_tp8_c6: {roles: {agg: {args: {max-num-seqs: 12}}}, benchmark: {env: {CONC: '6', KV_OFFLOADING: 'none'}}} -override_tp8_c8: {roles: {agg: {args: {max-num-seqs: 16}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: 'none'}}} -override_tp8_c10: {roles: {agg: {args: {max-num-seqs: 20}}}, benchmark: {env: {CONC: '10', KV_OFFLOADING: 'none'}}} -override_tp8_c16_lmcache: {roles: {agg: {args: {max-num-seqs: 32, kv-transfer-config: '{"kv_connector":"LMCacheMPConnector","kv_connector_module_path":"lmcache.integration.vllm.lmcache_mp_connector","kv_role":"kv_both","kv_connector_extra_config":{"lmcache.mp.server_urls":"tcp://127.0.0.1:5555,tcp://127.0.0.1:5556,tcp://127.0.0.1:5557,tcp://127.0.0.1:5558,tcp://127.0.0.1:5559,tcp://127.0.0.1:5560,tcp://127.0.0.1:5561,tcp://127.0.0.1:5562","lmcache.mp.mq_timeout":6000.0}}'}, env: {LMCACHE_SHARDS: '8', LMCACHE_L1_SHARD_GB: '162'}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '1298'}}} +override_tp8_c2: + roles: + agg: + args: + max-num-seqs: 4 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: 'none' + +override_tp8_c4: + roles: + agg: + args: + max-num-seqs: 8 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: 'none' + +override_tp8_c6: + roles: + agg: + args: + max-num-seqs: 12 + benchmark: + env: + CONC: '6' + KV_OFFLOADING: 'none' + +override_tp8_c8: + roles: + agg: + args: + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: 'none' + +override_tp8_c10: + roles: + agg: + args: + max-num-seqs: 20 + benchmark: + env: + CONC: '10' + KV_OFFLOADING: 'none' + +override_tp8_c16_lmcache: + roles: + agg: + args: + max-num-seqs: 32 + kv-transfer-config: '{"kv_connector":"LMCacheMPConnector","kv_connector_module_path":"lmcache.integration.vllm.lmcache_mp_connector","kv_role":"kv_both","kv_connector_extra_config":{"lmcache.mp.server_urls":"tcp://127.0.0.1:5555,tcp://127.0.0.1:5556,tcp://127.0.0.1:5557,tcp://127.0.0.1:5558,tcp://127.0.0.1:5559,tcp://127.0.0.1:5560,tcp://127.0.0.1:5561,tcp://127.0.0.1:5562","lmcache.mp.mq_timeout":6000.0}}' + env: + LMCACHE_SHARDS: '8' + LMCACHE_L1_SHARD_GB: '162' + benchmark: + env: + CONC: '16' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1298' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml index c4f69ae81b..43e25d9eec 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml @@ -63,12 +63,83 @@ base: AIPERF_APPLY_CHAT_TEMPLATE: 'true' # One variant per point. Admission is 2x CONC. -override_tp8_c1: {roles: {agg: {args: {max-num-seqs: 2}}}, benchmark: {env: {CONC: '1'}}} -override_tp8_c2: {roles: {agg: {args: {max-num-seqs: 4}}}, benchmark: {env: {CONC: '2'}}} -override_tp8_c4: {roles: {agg: {args: {max-num-seqs: 8}}}, benchmark: {env: {CONC: '4'}}} -override_tp8_c8: {roles: {agg: {args: {max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} -override_tp8_c10: {roles: {agg: {args: {max-num-seqs: 20}}}, benchmark: {env: {CONC: '10'}}} -override_tp8_c12: {roles: {agg: {args: {max-num-seqs: 24}}}, benchmark: {env: {CONC: '12'}}} -override_tp8_c14: {roles: {agg: {args: {max-num-seqs: 28}}}, benchmark: {env: {CONC: '14'}}} -override_tp8_c16: {roles: {agg: {args: {max-num-seqs: 32}}}, benchmark: {env: {CONC: '16'}}} -override_tp8_c18: {roles: {agg: {args: {max-num-seqs: 36}}}, benchmark: {env: {CONC: '18'}}} +override_tp8_c1: + roles: + agg: + args: + max-num-seqs: 2 + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + args: + max-num-seqs: 4 + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + args: + max-num-seqs: 8 + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + args: + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp8_c10: + roles: + agg: + args: + max-num-seqs: 20 + benchmark: + env: + CONC: '10' + +override_tp8_c12: + roles: + agg: + args: + max-num-seqs: 24 + benchmark: + env: + CONC: '12' + +override_tp8_c14: + roles: + agg: + args: + max-num-seqs: 28 + benchmark: + env: + CONC: '14' + +override_tp8_c16: + roles: + agg: + args: + max-num-seqs: 32 + benchmark: + env: + CONC: '16' + +override_tp8_c18: + roles: + agg: + args: + max-num-seqs: 36 + benchmark: + env: + CONC: '18' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agentic.yaml index 8ec01c32cf..d9c5e537a7 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agentic.yaml @@ -71,15 +71,134 @@ base: AIPERF_APPLY_CHAT_TEMPLATE: 'true' # One variant per point. Admission is 2x CONC. -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 2}}}, benchmark: {env: {CONC: '1'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 8}}}, benchmark: {env: {CONC: '4'}}} -override_tp4_c5: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 10}}}, benchmark: {env: {CONC: '5'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 16}}}, benchmark: {env: {CONC: '8'}}} -override_tp4_c10: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 20}}}, benchmark: {env: {CONC: '10'}}} -override_tp4_c12: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 24}}}, benchmark: {env: {CONC: '12'}}} -override_tp4_c15: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 30}}}, benchmark: {env: {CONC: '15'}}} -override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 40}}}, benchmark: {env: {CONC: '20'}}} -override_tp4_c24: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 48}}}, benchmark: {env: {CONC: '24'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-num-seqs: 64}}}, benchmark: {env: {CONC: '32'}}} -override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-num-seqs: 2}}}, benchmark: {env: {CONC: '1'}}} -override_tp2_c2: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-num-seqs: 4}}}, benchmark: {env: {CONC: '2'}}} +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 2 + benchmark: + env: + CONC: '1' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 8 + benchmark: + env: + CONC: '4' + +override_tp4_c5: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 10 + benchmark: + env: + CONC: '5' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp4_c10: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 20 + benchmark: + env: + CONC: '10' + +override_tp4_c12: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 24 + benchmark: + env: + CONC: '12' + +override_tp4_c15: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 30 + benchmark: + env: + CONC: '15' + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 40 + benchmark: + env: + CONC: '20' + +override_tp4_c24: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 48 + benchmark: + env: + CONC: '24' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 64 + benchmark: + env: + CONC: '32' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-num-seqs: 2 + benchmark: + env: + CONC: '1' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-num-seqs: 4 + benchmark: + env: + CONC: '2' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml index 900977903a..f31bfa382b 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml @@ -70,35 +70,522 @@ base: # races with TP2 HiCache shared-memory initialization. HiCache sizes each of # the target KV and Mamba host pools per rank with a 1 GB/rank alignment # reserve and the 1/15 NEXTN draft pool: (DRAM - TP) * 15 / TP / 31 GB. -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} -override_tp4_c12: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: none}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 32, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: none}}} -override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 40, cuda-graph-max-bs: 20}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: none}}} -override_tp4_c24: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 48, cuda-graph-max-bs: 24}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: none}}} -override_tp4_c28: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 56, cuda-graph-max-bs: 28}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: none}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 64, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: none}}} -override_tp4_c40: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 80, cuda-graph-max-bs: 40}}}, benchmark: {env: {CONC: '40', KV_OFFLOADING: none}}} -override_tp4_c48: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 96, cuda-graph-max-bs: 48}}}, benchmark: {env: {CONC: '48', KV_OFFLOADING: none}}} -override_tp4_c56: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 112, cuda-graph-max-bs: 56}}}, benchmark: {env: {CONC: '56', KV_OFFLOADING: none}}} -override_tp4_c60: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 120, cuda-graph-max-bs: 60}}}, benchmark: {env: {CONC: '60', KV_OFFLOADING: none}}} -override_tp4_c62: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 124, cuda-graph-max-bs: 62}}}, benchmark: {env: {CONC: '62', KV_OFFLOADING: none}}} -override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 128, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: none}}} -override_tp4_hicache_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 128, cuda-graph-max-bs: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_hicache_c66: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 132, cuda-graph-max-bs: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '66', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_hicache_c68: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 136, cuda-graph-max-bs: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '68', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_hicache_c70: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 140, cuda-graph-max-bs: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '70', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_hicache_c72: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 144, cuda-graph-max-bs: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '72', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} -override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} -override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} -override_tp2_c12: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: none}}} -override_tp2_c14: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 28, cuda-graph-max-bs: 14}}}, benchmark: {env: {CONC: '14', KV_OFFLOADING: none}}} -override_tp2_hicache_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 32, cuda-graph-max-bs: 16, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '432'}}} -override_tp2_hicache_c18: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 36, cuda-graph-max-bs: 18, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '18', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '432'}}} -override_tp2_hicache_c20: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 40, cuda-graph-max-bs: 20, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '432'}}} -override_tp2_hicache_c22: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 44, cuda-graph-max-bs: 22, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '22', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '432'}}} -override_tp2_hicache_c24: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 48, cuda-graph-max-bs: 24, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '432'}}} -override_tp2_hicache_c28: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 56, cuda-graph-max-bs: 28, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '432'}}} -override_tp2_hicache_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 64, cuda-graph-max-bs: 32, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '432'}}} +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp4_c12: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: none + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 32 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 40 + cuda-graph-max-bs: 20 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: none + +override_tp4_c24: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 48 + cuda-graph-max-bs: 24 + benchmark: + env: + CONC: '24' + KV_OFFLOADING: none + +override_tp4_c28: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 56 + cuda-graph-max-bs: 28 + benchmark: + env: + CONC: '28' + KV_OFFLOADING: none + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 64 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: none + +override_tp4_c40: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 80 + cuda-graph-max-bs: 40 + benchmark: + env: + CONC: '40' + KV_OFFLOADING: none + +override_tp4_c48: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 96 + cuda-graph-max-bs: 48 + benchmark: + env: + CONC: '48' + KV_OFFLOADING: none + +override_tp4_c56: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 112 + cuda-graph-max-bs: 56 + benchmark: + env: + CONC: '56' + KV_OFFLOADING: none + +override_tp4_c60: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 120 + cuda-graph-max-bs: 60 + benchmark: + env: + CONC: '60' + KV_OFFLOADING: none + +override_tp4_c62: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 124 + cuda-graph-max-bs: 62 + benchmark: + env: + CONC: '62' + KV_OFFLOADING: none + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 128 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '64' + KV_OFFLOADING: none + +override_tp4_hicache_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 128 + cuda-graph-max-bs: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '64' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c66: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 132 + cuda-graph-max-bs: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '66' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c68: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 136 + cuda-graph-max-bs: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '68' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c70: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 140 + cuda-graph-max-bs: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '70' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c72: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 144 + cuda-graph-max-bs: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '72' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp2_c12: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: none + +override_tp2_c14: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 28 + cuda-graph-max-bs: 14 + benchmark: + env: + CONC: '14' + KV_OFFLOADING: none + +override_tp2_hicache_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 32 + cuda-graph-max-bs: 16 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '16' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '432' + +override_tp2_hicache_c18: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 36 + cuda-graph-max-bs: 18 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '18' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '432' + +override_tp2_hicache_c20: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 40 + cuda-graph-max-bs: 20 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '20' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '432' + +override_tp2_hicache_c22: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 44 + cuda-graph-max-bs: 22 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '22' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '432' + +override_tp2_hicache_c24: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 48 + cuda-graph-max-bs: 24 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '24' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '432' + +override_tp2_hicache_c28: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 56 + cuda-graph-max-bs: 28 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '28' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '432' + +override_tp2_hicache_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 64 + cuda-graph-max-bs: 32 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '432' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml index 0b883dde79..817cebae91 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml @@ -69,36 +69,522 @@ base: # CONC, capped at 64. HiCache sizes each of the target KV and Mamba host pools # per rank with a 1 GB/rank alignment reserve and the 1/15 NEXTN draft pool: # (DRAM - TP) * 15 / TP / 31 GB. -override_tp8_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 2, cuda-graph-max-bs-decode: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 8, cuda-graph-max-bs-decode: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} -override_tp8_c8: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 16, cuda-graph-max-bs-decode: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} -override_tp8_c12: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 24, cuda-graph-max-bs-decode: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: none}}} -override_tp8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 32, cuda-graph-max-bs-decode: 16}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: none}}} -override_tp8_c20: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 40, cuda-graph-max-bs-decode: 20}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: none}}} -override_tp8_c24: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 48, cuda-graph-max-bs-decode: 24}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: none}}} -override_tp8_c28: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 56, cuda-graph-max-bs-decode: 28}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: none}}} -override_tp8_c32: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 64, cuda-graph-max-bs-decode: 32}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: none}}} -override_tp8_c40: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 80, cuda-graph-max-bs-decode: 40}}}, benchmark: {env: {CONC: '40', KV_OFFLOADING: none}}} -override_tp8_c48: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 96, cuda-graph-max-bs-decode: 48}}}, benchmark: {env: {CONC: '48', KV_OFFLOADING: none}}} -override_tp8_c56: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 112, cuda-graph-max-bs-decode: 56}}}, benchmark: {env: {CONC: '56', KV_OFFLOADING: none}}} -override_tp8_c60: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 120, cuda-graph-max-bs-decode: 60}}}, benchmark: {env: {CONC: '60', KV_OFFLOADING: none}}} -override_tp8_c62: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 124, cuda-graph-max-bs-decode: 62}}}, benchmark: {env: {CONC: '62', KV_OFFLOADING: none}}} -override_tp8_c64: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 128, cuda-graph-max-bs-decode: 64}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: none}}} -override_tp8_hicache_c64: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 128, cuda-graph-max-bs-decode: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} -override_tp8_hicache_c66: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 132, cuda-graph-max-bs-decode: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '66', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} -override_tp8_hicache_c68: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 136, cuda-graph-max-bs-decode: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '68', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} -override_tp8_hicache_c70: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 140, cuda-graph-max-bs-decode: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '70', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} -override_tp8_hicache_c72: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 144, cuda-graph-max-bs-decode: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '72', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} -override_tp8_hicache_c76: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, max-running-requests: 152, cuda-graph-max-bs-decode: 64, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '76', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '1731'}}} -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 2, cuda-graph-max-bs-decode: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 8, cuda-graph-max-bs-decode: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 16, cuda-graph-max-bs-decode: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} -override_tp4_c12: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 24, cuda-graph-max-bs-decode: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: none}}} -override_tp4_c14: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 28, cuda-graph-max-bs-decode: 14}}}, benchmark: {env: {CONC: '14', KV_OFFLOADING: none}}} -override_tp4_hicache_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 32, cuda-graph-max-bs-decode: 16, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_hicache_c18: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 36, cuda-graph-max-bs-decode: 18, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '18', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_hicache_c20: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 40, cuda-graph-max-bs-decode: 20, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_hicache_c22: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 44, cuda-graph-max-bs-decode: 22, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '22', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_hicache_c24: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 48, cuda-graph-max-bs-decode: 24, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_hicache_c28: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 56, cuda-graph-max-bs-decode: 28, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} -override_tp4_hicache_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, max-running-requests: 64, cuda-graph-max-bs-decode: 32, page-size: 64, enable-hierarchical-cache: true, hicache-size: 104, hicache-io-backend: kernel, hicache-mem-layout: page_first, hicache-write-policy: write_through_selective}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: dram, TOTAL_CPU_DRAM_GB: '865'}}} +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 2 + cuda-graph-max-bs-decode: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 8 + cuda-graph-max-bs-decode: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c8: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 16 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp8_c12: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 24 + cuda-graph-max-bs-decode: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: none + +override_tp8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 32 + cuda-graph-max-bs-decode: 16 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp8_c20: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 40 + cuda-graph-max-bs-decode: 20 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: none + +override_tp8_c24: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 48 + cuda-graph-max-bs-decode: 24 + benchmark: + env: + CONC: '24' + KV_OFFLOADING: none + +override_tp8_c28: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 56 + cuda-graph-max-bs-decode: 28 + benchmark: + env: + CONC: '28' + KV_OFFLOADING: none + +override_tp8_c32: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 64 + cuda-graph-max-bs-decode: 32 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: none + +override_tp8_c40: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 80 + cuda-graph-max-bs-decode: 40 + benchmark: + env: + CONC: '40' + KV_OFFLOADING: none + +override_tp8_c48: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 96 + cuda-graph-max-bs-decode: 48 + benchmark: + env: + CONC: '48' + KV_OFFLOADING: none + +override_tp8_c56: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 112 + cuda-graph-max-bs-decode: 56 + benchmark: + env: + CONC: '56' + KV_OFFLOADING: none + +override_tp8_c60: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 120 + cuda-graph-max-bs-decode: 60 + benchmark: + env: + CONC: '60' + KV_OFFLOADING: none + +override_tp8_c62: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 124 + cuda-graph-max-bs-decode: 62 + benchmark: + env: + CONC: '62' + KV_OFFLOADING: none + +override_tp8_c64: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 128 + cuda-graph-max-bs-decode: 64 + benchmark: + env: + CONC: '64' + KV_OFFLOADING: none + +override_tp8_hicache_c64: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 128 + cuda-graph-max-bs-decode: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '64' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_hicache_c66: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 132 + cuda-graph-max-bs-decode: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '66' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_hicache_c68: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 136 + cuda-graph-max-bs-decode: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '68' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_hicache_c70: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 140 + cuda-graph-max-bs-decode: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '70' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_hicache_c72: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 144 + cuda-graph-max-bs-decode: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '72' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_hicache_c76: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 152 + cuda-graph-max-bs-decode: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '76' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 2 + cuda-graph-max-bs-decode: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 8 + cuda-graph-max-bs-decode: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 16 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp4_c12: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 24 + cuda-graph-max-bs-decode: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: none + +override_tp4_c14: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 28 + cuda-graph-max-bs-decode: 14 + benchmark: + env: + CONC: '14' + KV_OFFLOADING: none + +override_tp4_hicache_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 32 + cuda-graph-max-bs-decode: 16 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '16' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c18: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 36 + cuda-graph-max-bs-decode: 18 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '18' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c20: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 40 + cuda-graph-max-bs-decode: 20 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '20' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c22: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 44 + cuda-graph-max-bs-decode: 22 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '22' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c24: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 48 + cuda-graph-max-bs-decode: 24 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '24' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c28: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 56 + cuda-graph-max-bs-decode: 28 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '28' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 64 + cuda-graph-max-bs-decode: 32 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml index 774646a201..0c2d2370cd 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml @@ -69,31 +69,431 @@ base: # race with HiCache shared-memory initialization. HiCache sizes the target KV # and Mamba host pools independently, and the NEXTN draft adds 1/15 of the # target: (599 GB - 1 GB/rank) * 15 / TP2 / 31 = 144 GB per pool. -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: 'none'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: 'none'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: 'none'}}} -override_tp4_c12: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: 'none'}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 32, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: 'none'}}} -override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 40, cuda-graph-max-bs: 20}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: 'none'}}} -override_tp4_c24: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 48, cuda-graph-max-bs: 24}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: 'none'}}} -override_tp4_c28: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 56, cuda-graph-max-bs: 28}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: 'none'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 64, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: 'none'}}} -override_tp4_c40: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 80, cuda-graph-max-bs: 40}}}, benchmark: {env: {CONC: '40', KV_OFFLOADING: 'none'}}} -override_tp4_c48: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 96, cuda-graph-max-bs: 48}}}, benchmark: {env: {CONC: '48', KV_OFFLOADING: 'none'}}} -override_tp4_c56: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 112, cuda-graph-max-bs: 56}}}, benchmark: {env: {CONC: '56', KV_OFFLOADING: 'none'}}} -override_tp4_c60: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 120, cuda-graph-max-bs: 60}}}, benchmark: {env: {CONC: '60', KV_OFFLOADING: 'none'}}} -override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 128, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: 'none'}}} -override_tp4_c68: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 136, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '68', KV_OFFLOADING: 'none'}}} -override_tp4_c72: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 144, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '72', KV_OFFLOADING: 'none'}}} -override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: 'none'}}} -override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: 'none'}}} -override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: 'none'}}} -override_tp2_c12: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: 'none'}}} -override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 32, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: 'none'}}} -override_tp2_c20: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 40, cuda-graph-max-bs: 20}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: 'none'}}} -override_tp2_c24: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 48, cuda-graph-max-bs: 24}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: 'none'}}} -override_tp2_c28: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 56, cuda-graph-max-bs: 28}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: 'none'}}} -override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 64, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: 'none'}}} -override_tp2_c36_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 72, cuda-graph-max-bs: 36, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '36', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} -override_tp2_c44_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 88, cuda-graph-max-bs: 44, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '44', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} -override_tp2_c52_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 104, cuda-graph-max-bs: 52, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '52', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: 'none' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: 'none' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: 'none' + +override_tp4_c12: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: 'none' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 32 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: 'none' + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 40 + cuda-graph-max-bs: 20 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: 'none' + +override_tp4_c24: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 48 + cuda-graph-max-bs: 24 + benchmark: + env: + CONC: '24' + KV_OFFLOADING: 'none' + +override_tp4_c28: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 56 + cuda-graph-max-bs: 28 + benchmark: + env: + CONC: '28' + KV_OFFLOADING: 'none' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 64 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: 'none' + +override_tp4_c40: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 80 + cuda-graph-max-bs: 40 + benchmark: + env: + CONC: '40' + KV_OFFLOADING: 'none' + +override_tp4_c48: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 96 + cuda-graph-max-bs: 48 + benchmark: + env: + CONC: '48' + KV_OFFLOADING: 'none' + +override_tp4_c56: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 112 + cuda-graph-max-bs: 56 + benchmark: + env: + CONC: '56' + KV_OFFLOADING: 'none' + +override_tp4_c60: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 120 + cuda-graph-max-bs: 60 + benchmark: + env: + CONC: '60' + KV_OFFLOADING: 'none' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 128 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '64' + KV_OFFLOADING: 'none' + +override_tp4_c68: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 136 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '68' + KV_OFFLOADING: 'none' + +override_tp4_c72: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 144 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '72' + KV_OFFLOADING: 'none' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: 'none' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: 'none' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: 'none' + +override_tp2_c12: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: 'none' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 32 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: 'none' + +override_tp2_c20: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 40 + cuda-graph-max-bs: 20 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: 'none' + +override_tp2_c24: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 48 + cuda-graph-max-bs: 24 + benchmark: + env: + CONC: '24' + KV_OFFLOADING: 'none' + +override_tp2_c28: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 56 + cuda-graph-max-bs: 28 + benchmark: + env: + CONC: '28' + KV_OFFLOADING: 'none' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 64 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: 'none' + +override_tp2_c36_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 72 + cuda-graph-max-bs: 36 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '36' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c44_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 88 + cuda-graph-max-bs: 44 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '44' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c52_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 104 + cuda-graph-max-bs: 52 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '52' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml index 913dcba2fe..274f545f90 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml @@ -68,37 +68,557 @@ base: # race with HiCache shared-memory initialization. HiCache sizes the target KV # and Mamba host pools independently, and the NEXTN draft adds 1/15 of the # target: (599 GB - 1 GB/rank) * 15 / TP2 / 31 = 144 GB per pool. -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: 'none'}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: 'none'}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: 'none'}}} -override_tp4_c12: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: 'none'}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 32, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: 'none'}}} -override_tp4_c20: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 40, cuda-graph-max-bs: 20}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: 'none'}}} -override_tp4_c24: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 48, cuda-graph-max-bs: 24}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: 'none'}}} -override_tp4_c28: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 56, cuda-graph-max-bs: 28}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: 'none'}}} -override_tp4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 64, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: 'none'}}} -override_tp4_c40: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 80, cuda-graph-max-bs: 40}}}, benchmark: {env: {CONC: '40', KV_OFFLOADING: 'none'}}} -override_tp4_c48: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 96, cuda-graph-max-bs: 48}}}, benchmark: {env: {CONC: '48', KV_OFFLOADING: 'none'}}} -override_tp4_c56: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 112, cuda-graph-max-bs: 56}}}, benchmark: {env: {CONC: '56', KV_OFFLOADING: 'none'}}} -override_tp4_c60: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 120, cuda-graph-max-bs: 60}}}, benchmark: {env: {CONC: '60', KV_OFFLOADING: 'none'}}} -override_tp4_c64: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 128, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '64', KV_OFFLOADING: 'none'}}} -override_tp4_c68: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 136, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '68', KV_OFFLOADING: 'none'}}} -override_tp4_c72: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, tokenizer-worker-num: 6, max-running-requests: 144, cuda-graph-max-bs: 64}}}, benchmark: {env: {CONC: '72', KV_OFFLOADING: 'none'}}} -override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: 'none'}}} -override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: 'none'}}} -override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: 'none'}}} -override_tp2_c12: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: 'none'}}} -override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 32, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: 'none'}}} -override_tp2_c20: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 40, cuda-graph-max-bs: 20}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: 'none'}}} -override_tp2_c24: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 48, cuda-graph-max-bs: 24}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: 'none'}}} -override_tp2_c28: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 56, cuda-graph-max-bs: 28}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: 'none'}}} -override_tp2_c32: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 64, cuda-graph-max-bs: 32}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: 'none'}}} -override_tp2_c32_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 64, cuda-graph-max-bs: 32, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} -override_tp2_c34_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 68, cuda-graph-max-bs: 34, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '34', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} -override_tp2_c36_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 72, cuda-graph-max-bs: 36, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '36', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} -override_tp2_c38_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 76, cuda-graph-max-bs: 38, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '38', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} -override_tp2_c40_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 80, cuda-graph-max-bs: 40, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '40', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} -override_tp2_c44_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 88, cuda-graph-max-bs: 44, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '44', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} -override_tp2_c48_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 96, cuda-graph-max-bs: 48, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '48', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} -override_tp2_c52_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 104, cuda-graph-max-bs: 52, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '52', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} -override_tp2_c56_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, max-running-requests: 112, cuda-graph-max-bs: 56, page-size: 64, enable-hierarchical-cache: true, hicache-size: 144, hicache-io-backend: 'kernel', hicache-mem-layout: 'page_first', hicache-write-policy: 'write_through_selective'}}}, benchmark: {env: {CONC: '56', KV_OFFLOADING: 'dram', TOTAL_CPU_DRAM_GB: '599'}}} +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: 'none' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: 'none' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: 'none' + +override_tp4_c12: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: 'none' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 32 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: 'none' + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 40 + cuda-graph-max-bs: 20 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: 'none' + +override_tp4_c24: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 48 + cuda-graph-max-bs: 24 + benchmark: + env: + CONC: '24' + KV_OFFLOADING: 'none' + +override_tp4_c28: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 56 + cuda-graph-max-bs: 28 + benchmark: + env: + CONC: '28' + KV_OFFLOADING: 'none' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 64 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: 'none' + +override_tp4_c40: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 80 + cuda-graph-max-bs: 40 + benchmark: + env: + CONC: '40' + KV_OFFLOADING: 'none' + +override_tp4_c48: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 96 + cuda-graph-max-bs: 48 + benchmark: + env: + CONC: '48' + KV_OFFLOADING: 'none' + +override_tp4_c56: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 112 + cuda-graph-max-bs: 56 + benchmark: + env: + CONC: '56' + KV_OFFLOADING: 'none' + +override_tp4_c60: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 120 + cuda-graph-max-bs: 60 + benchmark: + env: + CONC: '60' + KV_OFFLOADING: 'none' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 128 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '64' + KV_OFFLOADING: 'none' + +override_tp4_c68: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 136 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '68' + KV_OFFLOADING: 'none' + +override_tp4_c72: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 144 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '72' + KV_OFFLOADING: 'none' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: 'none' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: 'none' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: 'none' + +override_tp2_c12: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: 'none' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 32 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: 'none' + +override_tp2_c20: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 40 + cuda-graph-max-bs: 20 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: 'none' + +override_tp2_c24: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 48 + cuda-graph-max-bs: 24 + benchmark: + env: + CONC: '24' + KV_OFFLOADING: 'none' + +override_tp2_c28: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 56 + cuda-graph-max-bs: 28 + benchmark: + env: + CONC: '28' + KV_OFFLOADING: 'none' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 64 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: 'none' + +override_tp2_c32_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 64 + cuda-graph-max-bs: 32 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '32' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c34_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 68 + cuda-graph-max-bs: 34 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '34' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c36_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 72 + cuda-graph-max-bs: 36 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '36' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c38_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 76 + cuda-graph-max-bs: 38 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '38' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c40_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 80 + cuda-graph-max-bs: 40 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '40' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c44_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 88 + cuda-graph-max-bs: 44 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '44' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c48_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 96 + cuda-graph-max-bs: 48 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '48' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c52_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 104 + cuda-graph-max-bs: 52 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '52' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c56_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 112 + cuda-graph-max-bs: 56 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '56' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml index a58027c092..3c3cded953 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml @@ -64,13 +64,112 @@ base: # One variant per point. AgentX concurrency counts live session trees, so # admission is 2x CONC; the graph batch is CONC, capped at 64. -override_tp8_ep1_c4: {roles: {agg: {args: {expert-parallel-size: 1, cuda-graph-max-bs: 4, max-running-requests: 8}}}, benchmark: {env: {CONC: '4'}}} -override_tp8_ep1_c16: {roles: {agg: {args: {expert-parallel-size: 1, cuda-graph-max-bs: 16, max-running-requests: 32}}}, benchmark: {env: {CONC: '16'}}} -override_tp8_ep1_c20: {roles: {agg: {args: {expert-parallel-size: 1, cuda-graph-max-bs: 20, max-running-requests: 40}}}, benchmark: {env: {CONC: '20'}}} -override_tp8_ep1_c24: {roles: {agg: {args: {expert-parallel-size: 1, cuda-graph-max-bs: 24, max-running-requests: 48}}}, benchmark: {env: {CONC: '24'}}} -override_tp8_ep1_c32: {roles: {agg: {args: {expert-parallel-size: 1, cuda-graph-max-bs: 32, max-running-requests: 64}}}, benchmark: {env: {CONC: '32'}}} -override_tp8_ep8_c4: {roles: {agg: {args: {expert-parallel-size: 8, cuda-graph-max-bs: 4, max-running-requests: 8}}}, benchmark: {env: {CONC: '4'}}} -override_tp8_ep8_c16: {roles: {agg: {args: {expert-parallel-size: 8, cuda-graph-max-bs: 16, max-running-requests: 32}}}, benchmark: {env: {CONC: '16'}}} -override_tp8_ep8_c20: {roles: {agg: {args: {expert-parallel-size: 8, cuda-graph-max-bs: 20, max-running-requests: 40}}}, benchmark: {env: {CONC: '20'}}} -override_tp8_ep8_c24: {roles: {agg: {args: {expert-parallel-size: 8, cuda-graph-max-bs: 24, max-running-requests: 48}}}, benchmark: {env: {CONC: '24'}}} -override_tp8_ep8_c32: {roles: {agg: {args: {expert-parallel-size: 8, cuda-graph-max-bs: 32, max-running-requests: 64}}}, benchmark: {env: {CONC: '32'}}} +override_tp8_ep1_c4: + roles: + agg: + args: + expert-parallel-size: 1 + cuda-graph-max-bs: 4 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + +override_tp8_ep1_c16: + roles: + agg: + args: + expert-parallel-size: 1 + cuda-graph-max-bs: 16 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + +override_tp8_ep1_c20: + roles: + agg: + args: + expert-parallel-size: 1 + cuda-graph-max-bs: 20 + max-running-requests: 40 + benchmark: + env: + CONC: '20' + +override_tp8_ep1_c24: + roles: + agg: + args: + expert-parallel-size: 1 + cuda-graph-max-bs: 24 + max-running-requests: 48 + benchmark: + env: + CONC: '24' + +override_tp8_ep1_c32: + roles: + agg: + args: + expert-parallel-size: 1 + cuda-graph-max-bs: 32 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + +override_tp8_ep8_c4: + roles: + agg: + args: + expert-parallel-size: 8 + cuda-graph-max-bs: 4 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + +override_tp8_ep8_c16: + roles: + agg: + args: + expert-parallel-size: 8 + cuda-graph-max-bs: 16 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + +override_tp8_ep8_c20: + roles: + agg: + args: + expert-parallel-size: 8 + cuda-graph-max-bs: 20 + max-running-requests: 40 + benchmark: + env: + CONC: '20' + +override_tp8_ep8_c24: + roles: + agg: + args: + expert-parallel-size: 8 + cuda-graph-max-bs: 24 + max-running-requests: 48 + benchmark: + env: + CONC: '24' + +override_tp8_ep8_c32: + roles: + agg: + args: + expert-parallel-size: 8 + cuda-graph-max-bs: 32 + max-running-requests: 64 + benchmark: + env: + CONC: '32' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml index 2a36ff79cb..96e1d7b329 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml @@ -73,35 +73,479 @@ base: # SGLang reserves 15% of the static budget for the draft model: TP2 needs 0.95 # (an effective 0.8075) to fit weights, hybrid state and KV; TP4/TP8 use 0.80. # Tokenizer workers run only from TP4. -override_tp2_ep2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.95, cuda-graph-max-bs: 1, max-running-requests: 2}}}, benchmark: {env: {CONC: '1'}}} -override_tp2_ep2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.95, cuda-graph-max-bs: 4, max-running-requests: 8}}}, benchmark: {env: {CONC: '4'}}} -override_tp2_ep2_c6: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, expert-parallel-size: 2, mem-fraction-static: 0.95, cuda-graph-max-bs: 6, max-running-requests: 12}}}, benchmark: {env: {CONC: '6'}}} -override_tp4_ep1_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 1, max-running-requests: 2}}}, benchmark: {env: {CONC: '1'}}} -override_tp4_ep1_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 8, max-running-requests: 16}}}, benchmark: {env: {CONC: '8'}}} -override_tp4_ep1_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 16, max-running-requests: 32}}}, benchmark: {env: {CONC: '16'}}} -override_tp4_ep1_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 32, max-running-requests: 64}}}, benchmark: {env: {CONC: '32'}}} -override_tp4_ep1_c40: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 40, max-running-requests: 80}}}, benchmark: {env: {CONC: '40'}}} -override_tp4_ep1_c48: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 48, max-running-requests: 96}}}, benchmark: {env: {CONC: '48'}}} -override_tp4_ep4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 4, max-running-requests: 8}}}, benchmark: {env: {CONC: '4'}}} -override_tp4_ep4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 16, max-running-requests: 32}}}, benchmark: {env: {CONC: '16'}}} -override_tp4_ep4_c32: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 32, max-running-requests: 64}}}, benchmark: {env: {CONC: '32'}}} -override_tp4_ep4_c40: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 40, max-running-requests: 80}}}, benchmark: {env: {CONC: '40'}}} -override_tp4_ep4_c48: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, expert-parallel-size: 4, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 48, max-running-requests: 96}}}, benchmark: {env: {CONC: '48'}}} -override_tp8_ep1_c1: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 1, max-running-requests: 2}}}, benchmark: {env: {CONC: '1'}}} -override_tp8_ep1_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 4, max-running-requests: 8}}}, benchmark: {env: {CONC: '4'}}} -override_tp8_ep1_c8: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 8, max-running-requests: 16}}}, benchmark: {env: {CONC: '8'}}} -override_tp8_ep1_c12: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 12, max-running-requests: 24}}}, benchmark: {env: {CONC: '12'}}} -override_tp8_ep1_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 16, max-running-requests: 32}}}, benchmark: {env: {CONC: '16'}}} -override_tp8_ep1_c24: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 24, max-running-requests: 48}}}, benchmark: {env: {CONC: '24'}}} -override_tp8_ep1_c32: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 32, max-running-requests: 64}}}, benchmark: {env: {CONC: '32'}}} -override_tp8_ep1_c40: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 40, max-running-requests: 80}}}, benchmark: {env: {CONC: '40'}}} -override_tp8_ep1_c48: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 48, max-running-requests: 96}}}, benchmark: {env: {CONC: '48'}}} -override_tp8_ep1_c64: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 64, max-running-requests: 128}}}, benchmark: {env: {CONC: '64'}}} -override_tp8_ep1_c80: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 1, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 64, max-running-requests: 160}}}, benchmark: {env: {CONC: '80'}}} -override_tp8_ep8_c4: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 8, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 4, max-running-requests: 8}}}, benchmark: {env: {CONC: '4'}}} -override_tp8_ep8_c16: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 8, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 16, max-running-requests: 32}}}, benchmark: {env: {CONC: '16'}}} -override_tp8_ep8_c24: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 8, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 24, max-running-requests: 48}}}, benchmark: {env: {CONC: '24'}}} -override_tp8_ep8_c32: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 8, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 32, max-running-requests: 64}}}, benchmark: {env: {CONC: '32'}}} -override_tp8_ep8_c40: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 8, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 40, max-running-requests: 80}}}, benchmark: {env: {CONC: '40'}}} -override_tp8_ep8_c48: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 8, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 48, max-running-requests: 96}}}, benchmark: {env: {CONC: '48'}}} -override_tp8_ep8_c64: {roles: {agg: {gpus: 8, args: {tensor-parallel-size: 8, expert-parallel-size: 8, mem-fraction-static: 0.8, tokenizer-worker-num: 6, cuda-graph-max-bs: 64, max-running-requests: 128}}}, benchmark: {env: {CONC: '64'}}} +override_tp2_ep2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.95 + cuda-graph-max-bs: 1 + max-running-requests: 2 + benchmark: + env: + CONC: '1' + +override_tp2_ep2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.95 + cuda-graph-max-bs: 4 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + +override_tp2_ep2_c6: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.95 + cuda-graph-max-bs: 6 + max-running-requests: 12 + benchmark: + env: + CONC: '6' + +override_tp4_ep1_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 1 + max-running-requests: 2 + benchmark: + env: + CONC: '1' + +override_tp4_ep1_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 8 + max-running-requests: 16 + benchmark: + env: + CONC: '8' + +override_tp4_ep1_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 16 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + +override_tp4_ep1_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 32 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + +override_tp4_ep1_c40: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 40 + max-running-requests: 80 + benchmark: + env: + CONC: '40' + +override_tp4_ep1_c48: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 48 + max-running-requests: 96 + benchmark: + env: + CONC: '48' + +override_tp4_ep4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 4 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + +override_tp4_ep4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 16 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + +override_tp4_ep4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 32 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + +override_tp4_ep4_c40: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 40 + max-running-requests: 80 + benchmark: + env: + CONC: '40' + +override_tp4_ep4_c48: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 48 + max-running-requests: 96 + benchmark: + env: + CONC: '48' + +override_tp8_ep1_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 1 + max-running-requests: 2 + benchmark: + env: + CONC: '1' + +override_tp8_ep1_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 4 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + +override_tp8_ep1_c8: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 8 + max-running-requests: 16 + benchmark: + env: + CONC: '8' + +override_tp8_ep1_c12: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 12 + max-running-requests: 24 + benchmark: + env: + CONC: '12' + +override_tp8_ep1_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 16 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + +override_tp8_ep1_c24: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 24 + max-running-requests: 48 + benchmark: + env: + CONC: '24' + +override_tp8_ep1_c32: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 32 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + +override_tp8_ep1_c40: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 40 + max-running-requests: 80 + benchmark: + env: + CONC: '40' + +override_tp8_ep1_c48: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 48 + max-running-requests: 96 + benchmark: + env: + CONC: '48' + +override_tp8_ep1_c64: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 64 + max-running-requests: 128 + benchmark: + env: + CONC: '64' + +override_tp8_ep1_c80: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 64 + max-running-requests: 160 + benchmark: + env: + CONC: '80' + +override_tp8_ep8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 8 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 4 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + +override_tp8_ep8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 8 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 16 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + +override_tp8_ep8_c24: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 8 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 24 + max-running-requests: 48 + benchmark: + env: + CONC: '24' + +override_tp8_ep8_c32: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 8 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 32 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + +override_tp8_ep8_c40: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 8 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 40 + max-running-requests: 80 + benchmark: + env: + CONC: '40' + +override_tp8_ep8_c48: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 8 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 48 + max-running-requests: 96 + benchmark: + env: + CONC: '48' + +override_tp8_ep8_c64: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 8 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 64 + max-running-requests: 128 + benchmark: + env: + CONC: '64' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml index 7fc43af170..749c769773 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml @@ -66,20 +66,258 @@ base: # One variant per point. Admission is 2x CONC and the decode graph batch follows # it up to 128; TP4 adds tokenizer workers. HiCache holds 1.5x the device KV pool. -override_tp4_c1: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 2, cuda-graph-max-bs-decode: 2}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp4_c4: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 8, cuda-graph-max-bs-decode: 8}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} -override_tp4_c8: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 16, cuda-graph-max-bs-decode: 16}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} -override_tp4_c12: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 24, cuda-graph-max-bs-decode: 24}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: none}}} -override_tp4_c16: {roles: {agg: {gpus: 4, args: {tensor-parallel-size: 4, tokenizer-worker-num: 6, max-running-requests: 32, cuda-graph-max-bs-decode: 32}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: none}}} -override_tp2_c1: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 2, cuda-graph-max-bs-decode: 2}}}, benchmark: {env: {CONC: '1', KV_OFFLOADING: none}}} -override_tp2_c4: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 8, cuda-graph-max-bs-decode: 8}}}, benchmark: {env: {CONC: '4', KV_OFFLOADING: none}}} -override_tp2_c8: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 16, cuda-graph-max-bs-decode: 16}}}, benchmark: {env: {CONC: '8', KV_OFFLOADING: none}}} -override_tp2_c12: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 24, cuda-graph-max-bs-decode: 24}}}, benchmark: {env: {CONC: '12', KV_OFFLOADING: none}}} -override_tp2_c16: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 32, cuda-graph-max-bs-decode: 32}}}, benchmark: {env: {CONC: '16', KV_OFFLOADING: none}}} -override_tp2_c20: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 40, cuda-graph-max-bs-decode: 40}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: none}}} -override_tp2_c20_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 40, cuda-graph-max-bs-decode: 40, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: kernel, hicache-mem-layout: page_first}}}, benchmark: {env: {CONC: '20', KV_OFFLOADING: dram}}} -override_tp2_c24_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 48, cuda-graph-max-bs-decode: 48, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: kernel, hicache-mem-layout: page_first}}}, benchmark: {env: {CONC: '24', KV_OFFLOADING: dram}}} -override_tp2_c28_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 56, cuda-graph-max-bs-decode: 56, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: kernel, hicache-mem-layout: page_first}}}, benchmark: {env: {CONC: '28', KV_OFFLOADING: dram}}} -override_tp2_c32_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 64, cuda-graph-max-bs-decode: 64, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: kernel, hicache-mem-layout: page_first}}}, benchmark: {env: {CONC: '32', KV_OFFLOADING: dram}}} -override_tp2_c36_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 72, cuda-graph-max-bs-decode: 72, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: kernel, hicache-mem-layout: page_first}}}, benchmark: {env: {CONC: '36', KV_OFFLOADING: dram}}} -override_tp2_c40_hicache: {roles: {agg: {gpus: 2, args: {tensor-parallel-size: 2, max-running-requests: 80, cuda-graph-max-bs-decode: 80, enable-hierarchical-cache: true, hicache-ratio: 1.5, hicache-write-policy: write_through, hicache-io-backend: kernel, hicache-mem-layout: page_first}}}, benchmark: {env: {CONC: '40', KV_OFFLOADING: dram}}} +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp4_c12: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 24 + cuda-graph-max-bs-decode: 24 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: none + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp2_c12: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 24 + cuda-graph-max-bs-decode: 24 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: none + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp2_c20: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 40 + cuda-graph-max-bs-decode: 40 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: none + +override_tp2_c20_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 40 + cuda-graph-max-bs-decode: 40 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first + benchmark: + env: + CONC: '20' + KV_OFFLOADING: dram + +override_tp2_c24_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 48 + cuda-graph-max-bs-decode: 48 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first + benchmark: + env: + CONC: '24' + KV_OFFLOADING: dram + +override_tp2_c28_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 56 + cuda-graph-max-bs-decode: 56 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first + benchmark: + env: + CONC: '28' + KV_OFFLOADING: dram + +override_tp2_c32_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 64 + cuda-graph-max-bs-decode: 64 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + +override_tp2_c36_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 72 + cuda-graph-max-bs-decode: 72 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first + benchmark: + env: + CONC: '36' + KV_OFFLOADING: dram + +override_tp2_c40_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 80 + cuda-graph-max-bs-decode: 80 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first + benchmark: + env: + CONC: '40' + KV_OFFLOADING: dram diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml index 5ca678e34c..f355e0b994 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml @@ -64,7 +64,47 @@ base: # One variant per point. NEXTN resets an unset admission cap to 48, so it is # explicit: 2x CONC, with the graph batch equal to it. C16 needs a larger # static fraction and a bounded Mamba cache. -override_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 2, mem-fraction-static: 0.8}}}, benchmark: {env: {CONC: '1'}}} -override_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 8, mem-fraction-static: 0.8}}}, benchmark: {env: {CONC: '4'}}} -override_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 16, mem-fraction-static: 0.8}}}, benchmark: {env: {CONC: '8'}}} -override_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 32, mem-fraction-static: 0.9, max-mamba-cache-size: 160}}}, benchmark: {env: {CONC: '16'}}} +override_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 2 + mem-fraction-static: 0.8 + benchmark: + env: + CONC: '1' + +override_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 8 + mem-fraction-static: 0.8 + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 16 + mem-fraction-static: 0.8 + benchmark: + env: + CONC: '8' + +override_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 32 + mem-fraction-static: 0.9 + max-mamba-cache-size: 160 + benchmark: + env: + CONC: '16' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml index 5ba1dd8652..a15f04fc80 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml @@ -65,8 +65,52 @@ base: # One variant per point: admission is 2x CONC and the graph batch is CONC. NEXTN # resets an unset admission limit to 48, so it stays explicit. -override_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1'}}} -override_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4'}}} -override_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8'}}} -override_c12: {roles: {agg: {args: {max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12'}}} -override_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '16'}}} +override_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + +override_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + +override_c12: + roles: + agg: + args: + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + +override_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '16' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml index 76e92bfd7e..d780dbe25b 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml @@ -54,8 +54,52 @@ base: WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_with_subagents_256k # One variant per point: admission is 2x CONC and the graph batch is CONC. -override_c1: {roles: {agg: {args: {max-running-requests: 2, cuda-graph-max-bs: 1}}}, benchmark: {env: {CONC: '1'}}} -override_c4: {roles: {agg: {args: {max-running-requests: 8, cuda-graph-max-bs: 4}}}, benchmark: {env: {CONC: '4'}}} -override_c8: {roles: {agg: {args: {max-running-requests: 16, cuda-graph-max-bs: 8}}}, benchmark: {env: {CONC: '8'}}} -override_c12: {roles: {agg: {args: {max-running-requests: 24, cuda-graph-max-bs: 12}}}, benchmark: {env: {CONC: '12'}}} -override_c16: {roles: {agg: {args: {max-running-requests: 32, cuda-graph-max-bs: 16}}}, benchmark: {env: {CONC: '16'}}} +override_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + +override_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + +override_c12: + roles: + agg: + args: + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + +override_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '16' From ff0859d36c67e35ca1d1584bccc0945f771d5763 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 14:23:16 -0500 Subject: [PATCH 100/105] fix(agentx): give the Qwen3.8-Flash-Next B300 SGLang server a two-hour health budget --- .../qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml index a15f04fc80..52879777ed 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml @@ -54,6 +54,11 @@ base: SGLANG_ENABLE_FLASHINFER_GEMM: 'true' # Outlast AIPerf's 300 s client connection pool. SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + # The checkpoint loads from the shared HF cache on Lustre, which took over + # 30 minutes on a cold node. + health_check: + max_attempts: 1440 + interval_seconds: 5 benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh From 2a88c4b23be68c9bb14cb3b293f640af3c72537d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 14:23:46 -0500 Subject: [PATCH 101/105] fix(srt): keep GLM-5.2's 150-step SWE-bench budget for single-node AgentX evals The legacy GLM-5.2 scripts raised SWEBENCH_AGENT_STEP_LIMIT to 150 for eval-only runs. srt-slurm's post-eval only sees the runner's environment, so the recipe cannot set it; srt_eval.sh does. --- benchmarks/single_node/srt_eval.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/benchmarks/single_node/srt_eval.sh b/benchmarks/single_node/srt_eval.sh index 33591203ed..4fe17dafcb 100644 --- a/benchmarks/single_node/srt_eval.sh +++ b/benchmarks/single_node/srt_eval.sh @@ -16,6 +16,11 @@ eval_args=() if [[ "$IS_AGENTIC" != 1 ]]; then check_env_vars MAX_MODEL_LEN eval_args=(--framework lm-eval) +elif [[ "${MODEL_PREFIX:-}" == glm5.2 ]]; then + # GLM-5.2's template defaults to maximum reasoning effort without + # chat_template_kwargs, which mini-swe-agent never passes; the heavy thinking + # exhausts the shared step budget. The recipe env does not reach post-eval. + export SWEBENCH_AGENT_STEP_LIMIT=150 fi export PORT="${1##*:}" if [[ ! "$PORT" =~ ^[1-9][0-9]*$ || "$IS_MULTINODE" != false ]]; then From 478d5c45de265d6c36f2eae5fb184608eeb6c89f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 14:35:09 -0500 Subject: [PATCH 102/105] fix(agentx): give the MiniMax-M3 B200 TRT-LLM server a two-hour health budget --- .../minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml index 870350371e..987e19242c 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml @@ -24,6 +24,10 @@ base: # Keep Prometheus without rc23's per-step timing collector and accept BFCL's # store=false field. setup_script: minimaxm3-trtllm-rc23.sh + # TP8 loads, autotunes and captures graphs past the 30-minute default. + health_check: + max_attempts: 1440 + interval_seconds: 5 roles: agg: nodes: 1 From 997e9f437fd90e19cea819aed9b022dc8fb3ed7a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 14:36:44 -0500 Subject: [PATCH 103/105] fix(b200): default the Qwen3.8-Flash-Next NVFP4 path when no pool setting names it --- runners/launch_b200-nscale-slurm.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index 8560740d7d..1c6c7afc04 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -138,10 +138,8 @@ elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/scratch/models/Kimi-K3" export SRT_SLURM_MODEL_PREFIX="kimik3" elif [[ $MODEL_PREFIX == "qwen3.8next" && $PRECISION == "fp4" ]]; then - check_env_vars MODEL_PATH - if [[ -n "${MODEL_PATH}" && -d "$MODEL_PATH" ]]; then - : - else + # No pool setting names this checkpoint; default to the node-local copy. + if [[ -z "${MODEL_PATH:-}" || ! -d "$MODEL_PATH" ]]; then export MODEL_PATH="/scratch/models/Qwen3.8-Flash-Next-NVFP4" fi export SRT_SLURM_MODEL_PREFIX="qwen3.8next-fp4" From 730ffb9b3cd35778b4e20d46572f968b2074a6b6 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 25 Sep 2026 16:48:59 -0500 Subject: [PATCH 104/105] chore(minimaxm3): drop the vLLM SimpleCPUOffload patch The MiniMax-M3 B200/B300 DRAM points no longer patch vLLM's SimpleCPUOffload worker. The pinned nightly is pruned, so these points cannot run until the image moves to a build that no longer needs the patch (see #3131, #3132, #3435). --- .../configs/vllm-simple-kv-offload.sh | 4 - .../agentic/minimaxm3_fp4_b200_mtp.sh | 1 - .../agentic/minimaxm3_fp4_b300_mtp.sh | 1 - .../minimaxm3/vllm/b200-fp4-mtp/agentic.yaml | 12 +- .../minimaxm3/vllm/b300-fp4-mtp/agentic.yaml | 5 +- runners/patch_vllm_simple_kv_offload.py | 133 ------------------ 6 files changed, 2 insertions(+), 154 deletions(-) delete mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/vllm-simple-kv-offload.sh delete mode 100755 runners/patch_vllm_simple_kv_offload.py diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-simple-kv-offload.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-simple-kv-offload.sh deleted file mode 100755 index 208464daa7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-simple-kv-offload.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -# Split heterogeneous KV backing storage into valid SimpleCPUOffload regions in the worker's vLLM. -set -euo pipefail -python3 /infmax-workspace/runners/patch_vllm_simple_kv_offload.py diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh index ce28224f88..954007c8f0 100755 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh @@ -103,7 +103,6 @@ install_agentic_deps OFFLOAD_ARGS=() if require_agentic_kv_offload_backend vllm-simple; then - python3 "$(dirname "$0")/../../../runners/patch_vllm_simple_kv_offload.py" CPU_OFFLOAD_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024)) export VLLM_USE_SIMPLE_KV_OFFLOAD=1 OFFLOAD_CONFIG=$(printf \ diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh index c606792017..508c0d0359 100755 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh @@ -40,7 +40,6 @@ install_agentic_deps OFFLOAD_ARGS=() if require_agentic_kv_offload_backend vllm-simple; then - python3 "$(dirname "$0")/../../../runners/patch_vllm_simple_kv_offload.py" CPU_OFFLOAD_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024)) export VLLM_USE_SIMPLE_KV_OFFLOAD=1 OFFLOAD_CONFIG=$(printf \ diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml index 9a12664c47..901ea87ea6 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml @@ -60,8 +60,7 @@ base: MODEL: nvidia/MiniMax-M3-NVFP4 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' -# One variant per point. DRAM points patch vLLM's SimpleCPUOffload worker for -# heterogeneous layer regions and give it the whole host budget +# One variant per point. DRAM points give SimpleCPUOffload the whole host budget # (TOTAL_CPU_DRAM_GB GiB) with lazy offload. override_tp8_c1: roles: @@ -130,7 +129,6 @@ override_tp4_c20: KV_OFFLOADING: none override_tp4_c15_dram: - setup_script: vllm-simple-kv-offload.sh roles: agg: gpus: 4 @@ -146,7 +144,6 @@ override_tp4_c15_dram: TOTAL_CPU_DRAM_GB: '1082' override_tp4_c20_dram: - setup_script: vllm-simple-kv-offload.sh roles: agg: gpus: 4 @@ -162,7 +159,6 @@ override_tp4_c20_dram: TOTAL_CPU_DRAM_GB: '1082' override_tp4_c25_dram: - setup_script: vllm-simple-kv-offload.sh roles: agg: gpus: 4 @@ -178,7 +174,6 @@ override_tp4_c25_dram: TOTAL_CPU_DRAM_GB: '1082' override_tp4_c30_dram: - setup_script: vllm-simple-kv-offload.sh roles: agg: gpus: 4 @@ -194,7 +189,6 @@ override_tp4_c30_dram: TOTAL_CPU_DRAM_GB: '1082' override_tp4_c32_dram: - setup_script: vllm-simple-kv-offload.sh roles: agg: gpus: 4 @@ -210,7 +204,6 @@ override_tp4_c32_dram: TOTAL_CPU_DRAM_GB: '1082' override_tp4_c34_dram: - setup_script: vllm-simple-kv-offload.sh roles: agg: gpus: 4 @@ -226,7 +219,6 @@ override_tp4_c34_dram: TOTAL_CPU_DRAM_GB: '1082' override_tp4_c36_dram: - setup_script: vllm-simple-kv-offload.sh roles: agg: gpus: 4 @@ -242,7 +234,6 @@ override_tp4_c36_dram: TOTAL_CPU_DRAM_GB: '1082' override_tp4_c38_dram: - setup_script: vllm-simple-kv-offload.sh roles: agg: gpus: 4 @@ -258,7 +249,6 @@ override_tp4_c38_dram: TOTAL_CPU_DRAM_GB: '1082' override_tp4_c40_dram: - setup_script: vllm-simple-kv-offload.sh roles: agg: gpus: 4 diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml index 5913cada03..d775524820 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml @@ -60,8 +60,7 @@ base: MODEL: nvidia/MiniMax-M3-NVFP4 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' -# One variant per point. DRAM points patch vLLM's SimpleCPUOffload worker for -# heterogeneous layer regions and give it the whole host budget +# One variant per point. DRAM points give SimpleCPUOffload the whole host budget # (TOTAL_CPU_DRAM_GB GiB) with lazy offload. override_tp8_c1: roles: @@ -130,7 +129,6 @@ override_tp4_c20: KV_OFFLOADING: none override_tp4_c30_dram: - setup_script: vllm-simple-kv-offload.sh roles: agg: gpus: 4 @@ -146,7 +144,6 @@ override_tp4_c30_dram: TOTAL_CPU_DRAM_GB: '1024' override_tp2_c24_dram: - setup_script: vllm-simple-kv-offload.sh roles: agg: gpus: 2 diff --git a/runners/patch_vllm_simple_kv_offload.py b/runners/patch_vllm_simple_kv_offload.py deleted file mode 100755 index 872bd9960a..0000000000 --- a/runners/patch_vllm_simple_kv_offload.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python3 -"""Split heterogeneous vLLM KV backing storage into valid CPU offload regions.""" - -from __future__ import annotations - -import importlib.util -import sys -from pathlib import Path - -OLD_SETUP = """ logical_storage_bytes = self.kv_cache_config.kv_cache_tensors[0].size - - # The DMA backend copies whole blocks as base + block_id * stride(0), -""" -NEW_SETUP = """ logical_storage_bytes = self.kv_cache_config.kv_cache_tensors[0].size - split_storage_by_layer = any( - logical_storage_bytes - % (num_blocks * cache_tensor.block_stride) - != 0 - for cache_tensor in self.kv_cache_config.kv_cache_tensors - ) - - # The DMA backend copies whole blocks as base + block_id * stride(0), -""" -OLD_LOOP = """ unique_gpu_caches: dict[str, torch.Tensor] = {} - seen: set[tuple[torch.device, int]] = set() - for name, tensor in kv_caches.items(): - storage = tensor.untyped_storage() - key = (tensor.device, storage.data_ptr()) - if key in seen: - continue - seen.add(key) - - physical_per_block, remainder = divmod(tensor.shape[0], num_blocks) - assert remainder == 0, ( - f"KV cache {name!r} has {tensor.shape[0]} physical blocks, which " - f"is not divisible by {num_blocks} scheduler blocks" - ) - block_bytes = tensor.stride(0) * tensor.element_size() * physical_per_block - raw = torch.empty(0, dtype=torch.int8, device=tensor.device).set_(storage) - assert raw.numel() >= logical_storage_bytes, ( - f"KV cache {name!r} storage has {raw.numel()} bytes, smaller " - f"than the configured {logical_storage_bytes}-byte allocation" - ) - regions = raw[:logical_storage_bytes].view(-1, num_blocks, block_bytes) - for idx, region in enumerate(regions): - key_name = name if len(regions) == 1 else f"{name}.{idx}" - unique_gpu_caches[key_name] = region -""" -NEW_LOOP = """ unique_gpu_caches: dict[str, torch.Tensor] = {} - seen: set[tuple[torch.device, int, int, int]] = set() - for name, tensor in kv_caches.items(): - physical_per_block, remainder = divmod(tensor.shape[0], num_blocks) - assert remainder == 0, ( - f"KV cache {name!r} has {tensor.shape[0]} physical blocks, which " - f"is not divisible by {num_blocks} scheduler blocks" - ) - block_bytes = tensor.stride(0) * tensor.element_size() * physical_per_block - storage = tensor.untyped_storage() - raw = torch.empty(0, dtype=torch.int8, device=tensor.device).set_(storage) - - if split_storage_by_layer: - region_offset = tensor.storage_offset() * tensor.element_size() - region_bytes = num_blocks * block_bytes - else: - region_offset = 0 - region_bytes = logical_storage_bytes - - key = (tensor.device, storage.data_ptr(), region_offset, region_bytes) - if key in seen: - continue - seen.add(key) - - region_end = region_offset + region_bytes - assert raw.numel() >= region_end, ( - f"KV cache {name!r} storage has {raw.numel()} bytes, smaller " - f"than the required {region_end}-byte region" - ) - regions = raw[region_offset:region_end].view( - -1, num_blocks, block_bytes - ) - for idx, region in enumerate(regions): - key_name = name if len(regions) == 1 else f"{name}.{idx}" - unique_gpu_caches[key_name] = region -""" - - -def installed_worker_path() -> Path: - """Return the SimpleCPUOffload worker module from the installed vLLM.""" - spec = importlib.util.find_spec("vllm") - if spec is None or not spec.submodule_search_locations: - raise RuntimeError("vllm package is not installed") - package_root = Path(next(iter(spec.submodule_search_locations))) - return package_root / "v1/simple_kv_offload/worker.py" - - -def patch_worker(worker_path: Path) -> bool: - """Patch heterogeneous layer-region sizing and return whether source changed.""" - source = worker_path.read_text() - if NEW_SETUP in source and NEW_LOOP in source: - return False - if NEW_SETUP in source or NEW_LOOP in source: - raise RuntimeError(f"partially patched vLLM worker at {worker_path}") - if source.count(OLD_SETUP) != 1 or source.count(OLD_LOOP) != 1: - raise RuntimeError( - f"unsupported vLLM SimpleCPUOffload worker at {worker_path}" - ) - - patched = source.replace(OLD_SETUP, NEW_SETUP).replace(OLD_LOOP, NEW_LOOP) - worker_path.write_text(patched) - return True - - -def main(argv: list[str]) -> int: - if len(argv) > 2: - print(f"Usage: {argv[0]} [WORKER_PATH]", file=sys.stderr) - return 2 - - try: - worker_path = ( - Path(argv[1]).resolve() if len(argv) == 2 else installed_worker_path() - ) - changed = patch_worker(worker_path) - except (OSError, RuntimeError) as error: - print(f"ERROR: failed to patch vLLM CPU offload: {error}", file=sys.stderr) - return 1 - - state = "Patched" if changed else "Already patched" - print(f"{state} vLLM SimpleCPUOffload heterogeneous layer regions") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv)) From 8fc955179efc17dba02497e2730a9edc0f46a142 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 25 Sep 2026 23:34:53 -0500 Subject: [PATCH 105/105] style(srt): format single-node adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按 Ruff 格式化单节点适配器。 --- infx/srt_slurm/single_node.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/infx/srt_slurm/single_node.py b/infx/srt_slurm/single_node.py index b37eef516b..1e6baece4b 100644 --- a/infx/srt_slurm/single_node.py +++ b/infx/srt_slurm/single_node.py @@ -89,7 +89,9 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N raise ValueError(f"Unsupported single-node framework: {environment['FRAMEWORK']!r}") spec = spec_parameters(role, engine) if spec and spec["method"] not in {"eagle", "eagle3", "nextn", "mtp", "dspark"}: - raise ValueError("Single-node SRT supports only native MTP, EAGLE3, DSpark or no speculation") + raise ValueError( + "Single-node SRT supports only native MTP, EAGLE3, DSpark or no speculation" + ) # A point that stops drafting may keep its matrix label. speculation = "mtp" if spec else workload.get("SPEC_DECODING", "none") agentic = environment["IS_AGENTIC"] == "1" @@ -108,7 +110,9 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N # draft_model names a bundled or separate draft; its recipes speculate natively. "SPEC_DECODING": ( speculation, - "mtp" if environment["SPEC_DECODING"] == "draft_model" else environment["SPEC_DECODING"], + "mtp" + if environment["SPEC_DECODING"] == "draft_model" + else environment["SPEC_DECODING"], ), "AgentX client": (benchmark.get("command", "").endswith("srt_agentic.sh"), agentic), }