diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index a69ca7cc4d..0000000000 --- a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env bash - -# Agentic trace-replay recipe for a disaggregated SGLang server on MI355X -# (DeepSeek-V4-Pro FP4, 1P1D TP8). Driven by environment variables; submits a SLURM -# job via submit.sh. - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../../benchmark_lib.sh" --validation-only - -check_env_vars \ - TIME_LIMIT MODEL_PREFIX PRECISION RESULT_FILENAME DURATION \ - MAX_MODEL_LEN DISABLE_CUSTOM_ALL_REDUCE KV_OFFLOADING MORI_IO_SQ_BACKOFF_TIMEOUT_US \ - MORI_IO_QP_MAX_SEND_WR PREFILL_ROUTER_POLICY ENABLE_METRICS DECODE_MTP_SIZE - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - DURATION \ - KV_OFFLOADING \ - IS_AGENTIC \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -export MODEL_PREFIX -export PRECISION -export RESULT_FILENAME - -export DURATION -export MAX_MODEL_LEN - -# --disable-custom-all-reduce avoids a known aiter fault on MI355X. -export DISABLE_CUSTOM_ALL_REDUCE - -# ── KV cache offloading ── -# KV_OFFLOADING=none | dram (passed from YAML). -# KV_OFFLOAD_BACKEND selects the backend when offloading is on: -# hicache GPU + CPU-DRAM host pool (HICACHE_TIER L2), optionally + a -# Mooncake L3 store (HICACHE_TIER L3). The tunables below. -# umbp-linker UMBP as a DIRECT external store for the unified radix tree, -# with NO host cache tier in between. A different sglang code -# path, not a variation of HiCache -- sglang rejects the two -# together -- so it reads NONE of the HICACHE_*/MC_* tunables -# and takes UMBP_* instead (block further down). Implemented in -# amd_utils/server_sglang.sh; prefill-side only, like HiCache -# on this path, and dp-attn: true only. -export KV_OFFLOADING -if [[ "$KV_OFFLOADING" != "none" ]]; then - check_env_vars KV_OFFLOAD_BACKEND -fi -if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; then - check_env_vars \ - HICACHE_TIER HICACHE_HOST_POOL_COUNT HICACHE_PAGE_SIZE HICACHE_RATIO HICACHE_MEM_LAYOUT \ - HICACHE_IO_BACKEND HICACHE_WRITE_POLICY HICACHE_PREFETCH_POLICY MC_MASTER_PORT MC_METADATA_PORT \ - MC_METRICS_PORT MC_MASTER_THREADS MC_EVICTION_HIGH_WATERMARK MC_PROTOCOL \ - MC_GLOBAL_SEG - export HICACHE_TIER - export HICACHE_HOST_POOL_COUNT - # DSV4 uses page-size 256 (set in models.yaml); HiCache must match. - export HICACHE_PAGE_SIZE - export HICACHE_RATIO - # server_sglang.sh prefers --hicache-size over --hicache-ratio when TOTAL_CPU_DRAM_GB - # is set; opt out via FORCE_HICACHE_RATIO rather than unsetting TOTAL_CPU_DRAM_GB, - # which benchmark_lib.sh also requires client-side when KV_OFFLOADING=dram. - export FORCE_HICACHE_RATIO=1 - - if [[ "${HICACHE_TIER^^}" == "L3" ]]; then - export HICACHE_MEM_LAYOUT - export HICACHE_IO_BACKEND - export HICACHE_WRITE_POLICY - if [[ -z "${HICACHE_STORAGE_BACKEND:-}" ]]; then - export HICACHE_STORAGE_BACKEND=mooncake - fi - else - export HICACHE_MEM_LAYOUT - export HICACHE_IO_BACKEND - export HICACHE_WRITE_POLICY - export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" - fi - export HICACHE_PREFETCH_POLICY - # Shared nodes: use non-default Mooncake ports to avoid collisions. - export MC_MASTER_PORT - export MC_METADATA_PORT - export MC_METRICS_PORT - export MC_MASTER_THREADS - export MC_EVICTION_HIGH_WATERMARK - export MC_PROTOCOL - export MC_GLOBAL_SEG - export MC_DEVICE="${MC_DEVICE:-}" - export MC_MASTER_ADDR="${MC_MASTER_ADDR:-}" - export MC_METADATA_SERVER="${MC_METADATA_SERVER:-}" -fi - -# ── UMBP direct-linker tunables ── -# Only read when KV_OFFLOAD_BACKEND is a umbp-linker* arm. Defaults live in -# server_sglang.sh; these exports exist so the values are visible in the -# recipe (and in the commands dump) rather than buried, and so job.slurm has -# something to forward. -# UMBP_DRAM_BYTES NODE total for the tier, on the prefill node only. -# 1.5 TB matches the single-node linker arms, so a PD -# number can be read against them directly. Guarded in -# server_sglang.sh against UMBP_DRAM_CEILING_GB. -# UMBP_DRAM_CEILING_GB tier-size ceiling; defaults to half of MemTotal. -# UMBP_MAX_TOTAL_TOKENS optional device KV pool cap. UNSET on purpose: the -# linker is compared against the HiCache control at an -# IDENTICAL profiled pool, not at a capped one. -# UMBP_SA_WAIT_SECONDS ceiling for each of the three server-readiness waits -# (socket -> data plane -> host memory registered for -# GPU access). A 1.5 TB tier can take many minutes to -# register on a node holding a lot of page cache. -# UMBP_DRAM_USE_HUGEPAGES on by default; 2 MiB pages keep tier registration -# time predictable. The run fails rather than falling -# back to 4 KiB pages. Set 0 to opt out. -if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == umbp-linker* ]]; then - export UMBP_DRAM_BYTES="${UMBP_DRAM_BYTES:-1500000000000}" - export UMBP_DRAM_CEILING_GB="${UMBP_DRAM_CEILING_GB:-}" - export UMBP_DRAM_USE_HUGEPAGES="${UMBP_DRAM_USE_HUGEPAGES:-1}" - export UMBP_SA_WAIT_SECONDS="${UMBP_SA_WAIT_SECONDS:-1800}" - export UMBP_SA_WAIT_REGISTERED="${UMBP_SA_WAIT_REGISTERED:-1}" - export MORI_UMBP_LOG_LEVEL="${MORI_UMBP_LOG_LEVEL:-info}" - # Rank 0 opens the barrier only after the UMBP tier is registered, so the - # barrier must outlast UMBP_SA_WAIT_SECONDS. - export CONTAINER_BARRIER_TIMEOUT="${CONTAINER_BARRIER_TIMEOUT:-$((UMBP_SA_WAIT_SECONDS + 600))}" -fi - -# ── MoRIIO RDMA Send Queue tuning ── -export MORI_IO_SQ_BACKOFF_TIMEOUT_US -export MORI_IO_QP_MAX_SEND_WR - -export PREFILL_ROUTER_POLICY -export ENABLE_METRICS - -export DECODE_MTP_SIZE - -if [[ "${PREFILL_EP}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch the job. CONC_LIST is space-delimited in YAML; submit.sh wants 'x'. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/amd_utils/trace_replay.sh b/benchmarks/multi_node/amd_utils/trace_replay.sh index c1d5f76859..204e872cab 100644 --- a/benchmarks/multi_node/amd_utils/trace_replay.sh +++ b/benchmarks/multi_node/amd_utils/trace_replay.sh @@ -73,7 +73,7 @@ PORT="${ROUTER_PORT}" check_env_vars DURATION RESULT_FILENAME FLUSH_DRAIN_TIMEOUT CLEAR_CACHE_BETWEEN_CONC export MODEL DURATION MAX_MODEL_LEN # The workflow guard / upload steps expect one "${RESULT_FILENAME}_conc.json" per -# concurrency, so each conc below is suffixed with _conc (as agentic_srt.sh does). +# concurrency, so each conc below is suffixed with _conc (as srt_agentic.sh does). RESULT_FILENAME_BASE="${RESULT_FILENAME}" mkdir -p "$RESULT_DIR" @@ -103,7 +103,7 @@ for max_concurrency in "${chosen_concurrencies[@]}"; do # benchmark-multinode-tmpl.yml expects the per-conc nesting (LOGS/agentic/conc_*/...) # even though CI runs one concurrency per job; nesting also keeps local multi-conc - # sweeps from overwriting each other (same layout as agentic_srt.sh). + # sweeps from overwriting each other (same layout as srt_agentic.sh). CONC_RESULT_DIR="$RESULT_DIR/conc_${max_concurrency}" mkdir -p "$CONC_RESULT_DIR" diff --git a/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index a731508e05..0800750175 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -15,14 +15,14 @@ Store every recipe at `//-//` block holding only its differences (plain overrides, not `zip_override_*`). Master entries select one with `CONFIG_FILE=recipes//.yaml:override_`. Recipes read as text by a launcher, such as power recipes with top-level `telemetry:`, stay standalone. Keep distinct sweep entry files separate even when their contents match: recipe paths participate in eval grouping. The Qwen3.5 `*-stp-sweep.yaml` and `*-mtp-sweep.yaml` pair preserves that existing distinction. - Update `CONFIG_FILE` and `EVAL_CONFIG_FILE` references in active and deprecated master configs, launcher path rules, workflow filters, and local documentation together when moving a file. Preserve upstream source URLs as provenance and leave historical performance-changelog entries unchanged. No aliases for the old layout are provided. Shared runtime assets stay under `configs/` beside the model directories; they are not standalone recipes. The four files in `configs/dsv4-moe-load-balancer-configs/` are copied verbatim from NVIDIA/srt-slurm commit `deb1dfd9934398664f92d194169c183e009da83b`, preserving the EPLB initial expert assignments used by 17 DSV4 TRT recipes. `setup_srt_slurm()` stages them into the job checkout's `configs/` directory for the recipes' bind mounts. Keeping a recipe in this tree does not activate it; the master configs determine the benchmark matrix. diff --git a/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md b/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md index ef3c0ed034..27cd1c21c3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md +++ b/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md @@ -15,14 +15,14 @@ InferenceX 要求 srt-slurm 2.0 或更新版本，且配置必须声明 `schema: ```text dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml -qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml +qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml ``` - 使用主配置中的 `model-prefix` 和 `precision` 标签。引擎目录为 `sglang`、`vllm`、`trtllm` 或 `tilert`；前端仍在配置内显式声明。硬件目录使用 `b200`、`gb300` 等 GPU 型号，不使用集群名称。 - 工作负载目录为 `1k1k`、`8k1k` 或 `agentx`。已有的跨序列长度配置集合放在 `fixed-seq-len` 下，保留其覆盖项选择器。 - 文件名使用小写字母和连字符，以 `agg` 或 `disagg` 开头。包含拓扑及用于区分同目录配置的关键参数，例如并行方式、批大小、并发数、MTP、卸载或缓存设置。避免日期、带序号的延迟/吞吐量标签，以及重复目录中已有的模型或硬件信息。 - 拓扑名中的 `1p4d` 表示预填充/解码 worker 数，不一定等于物理节点数。`p-tp4` 和 `d-tp8` 分别标识预填充和解码 TP；`b` 表示批大小，`c` 表示并发数。运行参数以 YAML 为准。 -- 覆盖项集合使用 `*-variants.yaml` 命名。即使内容相同，也保留独立扫描入口：配置路径参与评估分组。Qwen3.5 的 `*-stp-sweep.yaml` 和 `*-mtp-sweep.yaml` 保留了这一既有区别。 +- 覆盖项集合使用 `*-variants.yaml` 命名。仅在各配置间存在差异的多节点 AgentX 配置，按主配置条目合并为一个集合，通常为 `agg-variants.yaml` 或 `disagg-variants.yaml`：`base` 保存共享设置，每个原配置成为一个具名 `override_` 块，只包含其差异（使用普通覆盖项，而非 `zip_override_*`）。主配置通过 `CONFIG_FILE=recipes//.yaml:override_` 选择其一。启动器以文本方式读取的配置（例如带顶层 `telemetry:` 的功耗配置）保持独立文件。即使内容相同，也保留独立扫描入口：配置路径参与评估分组。Qwen3.5 的 `*-stp-sweep.yaml` 和 `*-mtp-sweep.yaml` 保留了这一既有区别。 - 移动文件时，同步更新当前及已弃用主配置中的 `CONFIG_FILE`、`EVAL_CONFIG_FILE`，以及启动器路径规则、工作流过滤器和本地文档。保留上游来源 URL，并保持历史性能变更日志不变。不为旧目录结构提供别名。 共享运行时资源保留在模型目录旁的 `configs/` 中，不属于独立基准测试配置。`configs/dsv4-moe-load-balancer-configs/` 中的四个文件原样取自 NVIDIA/srt-slurm 提交 `deb1dfd9934398664f92d194169c183e009da83b`，保留了 17 个 DSV4 TRT 配置使用的 EPLB 初始专家分配。`setup_srt_slurm()` 将这些文件复制到作业仓库的 `configs/` 目录，供配置中的绑定挂载使用。将配置文件放入本目录不会启用该配置；实际基准测试矩阵由主配置决定。 diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h100-block32.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h100-block32.sh new file mode 100755 index 0000000000..4332962a39 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h100-block32.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Install the measured H100 DeepSeek-V4.1-Flash block-32 tilings into the worker's SGLang. +set -euo pipefail +agentic=/infmax-workspace/benchmarks/single_node/agentic +python3 "$agentic/install_h100_block32_configs.py" "$agentic/kernel_configs/h100_dsv41_block32" /logs diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h200-block32.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h200-block32.sh new file mode 100755 index 0000000000..3d311f530c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h200-block32.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Install the measured H200 DeepSeek-V4.1-Flash block-32 tilings into the worker's SGLang. +set -euo pipefail +agentic=/infmax-workspace/benchmarks/single_node/agentic +python3 "$agentic/install_h200_block32_configs.py" "$agentic/kernel_configs/h200_dsv41_block32" /logs "$DSV41_BLOCK32_TP" diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-atom-dspark-draft.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-atom-dspark-draft.sh new file mode 100644 index 0000000000..7419f48b64 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-atom-dspark-draft.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Stage the Kimi-K3 DSpark draft before ATOM starts: with an uncached repo id +# every rank pulls the same 7 GB at once. Shared-cache downloads can hit +# transient stale handles, hence the retries. +set -euo pipefail +unset HTTP_PROXY HTTPS_PROXY http_proxy https_proxy +for attempt in 1 2 3 4 5; do + hf download Inferact/Kimi-K3-DSpark && exit 0 + echo "hf download attempt $attempt failed; retrying in 60s" >&2 + sleep 60 +done +echo "hf download of Inferact/Kimi-K3-DSpark failed after 5 attempts" >&2 +exit 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh new file mode 100755 index 0000000000..243b3d2111 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Pin the worker's Mooncake client and point its store at one active RDMA rail. +set -euo pipefail +pip_install=(python3 -m pip install) +if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages; then + pip_install+=(--break-system-packages) +fi +"${pip_install[@]}" --quiet --no-cache-dir --no-deps --force-reinstall \ + mooncake-transfer-engine-cuda13==0.3.11.post1 +python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null + +# Rail-isolated nodes: two RNICs cannot reach each other even within a node, so +# every rank uses one rail. mlx5_0 is down on some nodes, and topology discovery +# then finds no HCA, so take the first active rail at runtime. DSXE nodes name +# their rails rdmap*. +rail="" +for device in mlx5_0 mlx5_1 mlx5_2 mlx5_3 mlx5_4 mlx5_5 mlx5_8 mlx5_9 \ + mlx5_10 mlx5_11 mlx5_16 mlx5_17 mlx5_20 mlx5_21 mlx5_22 mlx5_23 \ + $(ls /sys/class/infiniband 2>/dev/null | grep '^rdmap' | sort -V); do + if grep -q ACTIVE "/sys/class/infiniband/$device/ports/1/state" 2>/dev/null; then + rail="$device" + break + fi +done +if [[ -z "$rail" ]]; then + echo "Error: no active RDMA rail on $(hostname); Mooncake cannot initialise" >&2 + for state in /sys/class/infiniband/*/ports/*/state; do + echo "$state: $(cat "$state" 2>&1)" >&2 + done + exit 1 +fi +config="${MOONCAKE_CONFIG_PATH:-/logs/mooncake_store_config.json}" +python3 - "$config" "$rail" <<'PY' +import json, sys +path, rail = sys.argv[1:] +with open(path) as handle: + config = json.load(handle) +config["device_name"] = rail +with open(path, "w") as handle: + json.dump(config, handle, indent=2) +PY +echo "Mooncake rail: $rail" diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/lmcache-mp-rocm.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/lmcache-mp-rocm.sh new file mode 100755 index 0000000000..a69ec274b1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/lmcache-mp-rocm.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Start one LMCache MP server per TP rank in the worker container before vLLM, +# as the legacy MI300X MiniMax-M3 AgentX script did. A variant opts in with +# LMCACHE_SHARDS and LMCACHE_L1_SHARD_GB; its kv-transfer-config lists +# tcp://127.0.0.1:5555 through 5555 + LMCACHE_SHARDS - 1. +set -euo pipefail +[[ -n "${LMCACHE_SHARDS:-}" ]] || exit 0 +: "${LMCACHE_L1_SHARD_GB:?}" +version=0.5.3 +pip_install=(python3 -m pip install) +if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages; then + pip_install+=(--break-system-packages) +fi +"${pip_install[@]}" --quiet --no-cache-dir --no-deps \ + "sortedcontainers==2.4.0" \ + "opentelemetry-exporter-prometheus==0.61b0" \ + "cupy-rocm-7-0==14.1.1" \ + "lmcache==${version}" \ + --find-links "https://github.com/LMCache/LMCache/releases/expanded_assets/v${version}-rocm" +python3 -c "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" + +pids=() +for ((shard = 0; shard < LMCACHE_SHARDS; shard++)); do + # Detached so the servers outlive this preamble and serve the vLLM step. + setsid lmcache server \ + --host 127.0.0.1 --port $((5555 + shard)) \ + --http-host 127.0.0.1 --http-port $((8080 + shard)) \ + --l1-size-gb "$LMCACHE_L1_SHARD_GB" --l1-init-size-gb 10 \ + --l1-read-ttl-seconds 7200 --chunk-size 256 --max-workers 2 \ + --eviction-policy LRU --supported-transfer-mode lmcache_driven \ + > "/logs/lmcache_server_${shard}.log" 2>&1 < /dev/null & + pids+=($!) +done +for ((shard = 0; shard < LMCACHE_SHARDS; shard++)); do + for ((attempt = 0; ; attempt++)); do + python3 -c 'import sys, urllib.request; urllib.request.urlopen(sys.argv[1], timeout=2)' \ + "http://127.0.0.1:$((8080 + shard))/healthcheck" 2> /dev/null && break + if ! kill -0 "${pids[$shard]}" 2>/dev/null || (( attempt >= 600 )); then + echo "ERROR: LMCache server $shard did not become ready" >&2 + tail -n 50 "/logs/lmcache_server_${shard}.log" >&2 || true + exit 1 + fi + sleep 1 + done +done +echo "LMCache: ${LMCACHE_SHARDS} servers ready, ${LMCACHE_L1_SHARD_GB} GB L1 each" diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-atom-eagle3-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-atom-eagle3-deps.sh new file mode 100644 index 0000000000..eb8f9016aa --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-atom-eagle3-deps.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Stage the MiniMax-M3 EAGLE3 GQA draft and the tokenizer dependencies the ATOM +# image lacks, in a side directory that the server's interpreter imports. +set -euo pipefail +hf download Inferact/MiniMax-M3-EAGLE3-GQA +deps=/tmp/inferencex-atom-runtime-deps +/opt/venv/bin/python -m pip install --quiet --target "$deps" --no-deps sentencepiece tiktoken +site=$(/opt/venv/bin/python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') +echo "$deps" > "$site/inferencex-atom-runtime-deps.pth" diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-trtllm-agentx.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-trtllm-agentx.sh new file mode 100755 index 0000000000..d84e9c2787 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-trtllm-agentx.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Prepare a TensorRT-LLM 1.3 worker for MiniMax-M3 AgentX: keep request and +# iteration metrics without rc23's per-step timing collector, and accept +# OpenAI's store=false chat field. Every rank runs this; the lock serializes +# ranks that share a container, and each step is a no-op once applied. +set -euo pipefail +exec 9>/tmp/minimaxm3-trtllm-agentx.lock +flock 9 +root=$(python3 -c 'from importlib.util import find_spec; from pathlib import Path; print(Path(find_spec("tensorrt_llm").origin).parent)') +executor="$root/_torch/pyexecutor/py_executor.py" +gate="enabled=getattr(self.llm_args, 'return_perf_metrics', False))" +if grep -Fq "$gate" "$executor"; then + sed -i "s/enabled=getattr(self.llm_args, 'return_perf_metrics', False))/enabled=False)/" "$executor" +fi +grep -Fq "self.perf_manager = PerfMetricsManager(" "$executor" +grep -Fq "enabled=False)" "$executor" +python3 /infmax-workspace/runners/patch_trtllm_chat_store.py diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-trtllm-rc23.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-trtllm-rc23.sh new file mode 100755 index 0000000000..ed18542140 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/minimaxm3-trtllm-rc23.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Patch the worker's TensorRT-LLM 1.3.0rc23 for MiniMax-M3 AgentX: keep +# Prometheus request/iteration metrics without the per-step timing collector, +# and accept BFCL's standard store=false chat field. Every MPI rank runs this +# in the node's container, so serialize and skip what is already applied. +set -eo pipefail +ws=/infmax-workspace +exec 9>/tmp/minimaxm3-trtllm-rc23.lock +flock 9 +IS_AGENTIC=0 SCENARIO_TYPE='' source "$ws/benchmarks/benchmark_lib.sh" +py_executor="$(python3 -c 'from importlib.util import find_spec; from pathlib import Path; print(Path(find_spec("tensorrt_llm").origin).parent)')/_torch/pyexecutor/py_executor.py" +if ! grep -Fq "enabled=False)" "$py_executor"; then + disable_trtllm_detailed_perf_metrics +fi +python3 "$ws/runners/patch_trtllm_chat_store.py" diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/pip-runtime-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/pip-runtime-deps.sh new file mode 100755 index 0000000000..a67df2bc04 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/pip-runtime-deps.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Install packages an image lacks before its process starts. The worker role or +# the frontend lists them in SETUP_PIP_PACKAGES; empty installs nothing. +set -euo pipefail +[[ -n "${SETUP_PIP_PACKAGES:-}" ]] || exit 0 +pip_install=(python3 -m pip install --quiet) +if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages; then + pip_install+=(--break-system-packages) +fi +read -r -a packages <<< "$SETUP_PIP_PACKAGES" +"${pip_install[@]}" "${packages[@]}" diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/sglang-multi-tokenizer-cached-tokens-details.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/sglang-multi-tokenizer-cached-tokens-details.sh new file mode 100755 index 0000000000..e0c0fb4a73 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/sglang-multi-tokenizer-cached-tokens-details.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Forward cached_tokens_details through SGLang's multi-tokenizer path, which +# older releases drop, so AIPerf sees per-request cache hits. +set -euo pipefail +file=/sgl-workspace/sglang/python/sglang/srt/managers/multi_tokenizer_mixin.py +if ! sed -n '/elif isinstance(output, BatchStrOutput):/,/input_token_logprobs_val=_extract_field_by_index/p' "$file" \ + | grep -q 'cached_tokens_details=_extract_field_by_index'; then + sed -i '/elif isinstance(output, BatchStrOutput):/,/input_token_logprobs_val=_extract_field_by_index/ { + /cached_tokens=_extract_field_by_index(output, "cached_tokens", i),/a\ + cached_tokens_details=_extract_field_by_index(\ + output, "cached_tokens_details", i\ + ), + }' "$file" +fi diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-mooncake-0.3.11.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-mooncake-0.3.11.sh new file mode 100755 index 0000000000..74b8c63c3c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-mooncake-0.3.11.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Pin the worker's Mooncake store client to the release the master runs. +set -euo pipefail +pip_install=(python3 -m pip install) +if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages; then + pip_install+=(--break-system-packages) +fi +"${pip_install[@]}" --quiet --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 +python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-router-0.1.14.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-router-0.1.14.sh new file mode 100755 index 0000000000..9b372a0478 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-router-0.1.14.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Install the vLLM Router that fronts single-node DP-attention ranks (worker and router containers). +set -euo pipefail +pip_install=(python3 -m pip install) +if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages; then + pip_install+=(--break-system-packages) +fi +"${pip_install[@]}" --quiet vllm-router==0.1.14 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c1-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c1-mtp-hicache.yaml deleted file mode 100644 index f2368fa8a8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c1-mtp-hicache.yaml +++ /dev/null @@ -1,136 +0,0 @@ -schema: 2 -name: "agg-b200-tp8-c1-mtp-hicache" - -# B200 AgentX aggregate topology: one TP8 worker uses one eight-GPU node -# and serves both prefill and decode with bundled DSpark and HiCache. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" - frameworks: - dynamo: "1.5.0.dev20260914" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "b200" - gpus_per_node: 8 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 8 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: high - PIP_BREAK_SYSTEM_PACKAGES: "1" - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" - enable-metrics: true - enable-cache-report: true - trust-remote-code: true - weight-loader-prefetch-checkpoints: true - stream-interval: 10 - watchdog-timeout: 1000000 - mem-fraction-static: 0.90 - page-size: 256 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - moe-runner-backend: "flashinfer_mxfp4" - enable-deepseek-v4-fp4-indexer: true - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 2 - cuda-graph-max-bs-decode: 2 - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 8 - ep-size: 1 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-ratio: 2.75 - hicache-write-policy: write_through - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - -sbatch_directives: - mem: "0" - # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; - # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. - cpus-per-task: "192" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "false" - TP: "8" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c4-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c4-mtp-hicache.yaml deleted file mode 100644 index 5cc7a371c4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c4-mtp-hicache.yaml +++ /dev/null @@ -1,136 +0,0 @@ -schema: 2 -name: "agg-b200-tp8-c4-mtp-hicache" - -# B200 AgentX aggregate topology: one TP8 worker uses one eight-GPU node -# and serves both prefill and decode with bundled DSpark and HiCache. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" - frameworks: - dynamo: "1.5.0.dev20260914" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "b200" - gpus_per_node: 8 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 8 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: high - PIP_BREAK_SYSTEM_PACKAGES: "1" - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" - enable-metrics: true - enable-cache-report: true - trust-remote-code: true - weight-loader-prefetch-checkpoints: true - stream-interval: 10 - watchdog-timeout: 1000000 - mem-fraction-static: 0.90 - page-size: 256 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - moe-runner-backend: "flashinfer_mxfp4" - enable-deepseek-v4-fp4-indexer: true - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 8 - cuda-graph-max-bs-decode: 8 - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 8 - ep-size: 1 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-ratio: 2.75 - hicache-write-policy: write_through - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - -sbatch_directives: - mem: "0" - # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; - # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. - cpus-per-task: "192" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "false" - TP: "8" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c8-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c8-mtp-hicache.yaml deleted file mode 100644 index e1f28fa87d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c8-mtp-hicache.yaml +++ /dev/null @@ -1,136 +0,0 @@ -schema: 2 -name: "agg-b200-tp8-c8-mtp-hicache" - -# B200 AgentX aggregate topology: one TP8 worker uses one eight-GPU node -# and serves both prefill and decode with bundled DSpark and HiCache. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" - frameworks: - dynamo: "1.5.0.dev20260914" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "b200" - gpus_per_node: 8 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 8 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: high - PIP_BREAK_SYSTEM_PACKAGES: "1" - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" - enable-metrics: true - enable-cache-report: true - trust-remote-code: true - weight-loader-prefetch-checkpoints: true - stream-interval: 10 - watchdog-timeout: 1000000 - mem-fraction-static: 0.90 - page-size: 256 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - moe-runner-backend: "flashinfer_mxfp4" - enable-deepseek-v4-fp4-indexer: true - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 8 - ep-size: 1 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-ratio: 2.75 - hicache-write-policy: write_through - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - -sbatch_directives: - mem: "0" - # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; - # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. - cpus-per-task: "192" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "false" - TP: "8" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..3e302a5ac7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml @@ -0,0 +1,151 @@ +# AgentX dsv4 sglang b200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + # B200 AgentX aggregate topology: one TP8 worker uses one eight-GPU node + # and serves both prefill and decode with bundled DSpark and HiCache. + model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" + frameworks: + dynamo: "1.5.0.dev20260914" + dynamo: + install: true + source: + wheel: "1.5.0.dev20260914" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: "b200" + gpus_per_node: 8 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" + enable-metrics: true + enable-cache-report: true + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.90 + page-size: 256 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + enable-deepseek-v4-fp4-indexer: true + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 8 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + enable-hierarchical-cache: true + hicache-ratio: 2.75 + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + sbatch_directives: + mem: "0" + # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; + # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. + cpus-per-task: "192" + srun_options: + mem: "0" + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "8" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + +override_c1: + name: "agg-b200-tp8-c1-mtp-hicache" + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + +override_c4: + name: "agg-b200-tp8-c4-mtp-hicache" + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + +override_c8: + name: "agg-b200-tp8-c8-mtp-hicache" + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml deleted file mode 100644 index 305bf5d7ef..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml +++ /dev/null @@ -1,234 +0,0 @@ -schema: 2 -name: "disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload" - -# B200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 -# (1P x DEP8 / 1D x DEP8, bundled DSpark + HiCache KV offload), tuned for concurrency 128. -# Each DEP8 worker occupies one eight-GPU B200 node. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" - frameworks: - dynamo: "1.5.0.dev20260914" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: b200 - gpus_per_node: 8 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 8 - env: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - # P/D workers are on separate HGX B200 nodes, so use Mooncake's standard - # RDMA-registered CUDA buffers rather than an NVL72 custom memory pool. - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - # Use the eight external HCAs. mlx5_6..9 are on a separate low-LID - # fabric, while mlx5_bond_0 caused cross-fabric QP RTR timeouts. - disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - # Favor the full-attention pool; measured SWA utilization remained low. - swa-full-tokens-ratio: 0.01 - max-running-requests: 256 - cuda-graph-max-bs-decode: 256 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 8 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - - decode: - nodes: 1 - workers: 1 - gpus: 8 - - env: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_RAGGED_VERIFY_MODE: "static" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 256 - cuda-graph-max-bs-decode: 256 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; - # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. - cpus-per-task: "192" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml deleted file mode 100644 index 16f1ed6c8e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml +++ /dev/null @@ -1,234 +0,0 @@ -schema: 2 -name: "disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload" - -# B200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 -# (1P x DEP8 / 1D x DEP8, bundled DSpark + HiCache KV offload), tuned for concurrency 64. -# Each DEP8 worker occupies one eight-GPU B200 node. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" - frameworks: - dynamo: "1.5.0.dev20260914" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: b200 - gpus_per_node: 8 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 8 - env: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - # P/D workers are on separate HGX B200 nodes, so use Mooncake's standard - # RDMA-registered CUDA buffers rather than an NVL72 custom memory pool. - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - # Use the eight external HCAs. mlx5_6..9 are on a separate low-LID - # fabric, while mlx5_bond_0 caused cross-fabric QP RTR timeouts. - disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - # Favor the full-attention pool; measured SWA utilization remained low. - swa-full-tokens-ratio: 0.01 - max-running-requests: 256 - cuda-graph-max-bs-decode: 256 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 8 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - - decode: - nodes: 1 - workers: 1 - gpus: 8 - - env: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_RAGGED_VERIFY_MODE: "static" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 128 - cuda-graph-max-bs-decode: 256 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; - # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. - cpus-per-task: "192" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml deleted file mode 100644 index 4ac6c19f63..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml +++ /dev/null @@ -1,239 +0,0 @@ -schema: 2 -name: "disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload" - -# B200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 -# (2P x DEP8 / 1D x DEP8, bundled DSpark + HiCache KV offload), tuned for concurrency 256. -# Each DEP8 worker occupies one eight-GPU B200 node. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" - frameworks: - dynamo: "1.5.0.dev20260914" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: b200 - gpus_per_node: 8 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - prefill: - nodes: 2 - workers: 2 - gpus: 8 - env: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - # P/D workers are on separate HGX B200 nodes, so use Mooncake's standard - # RDMA-registered CUDA buffers rather than an NVL72 custom memory pool. - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_DSV4_MHC_PREWARM: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - # Use the eight external HCAs. mlx5_6..9 are on a separate low-LID - # fabric, while mlx5_bond_0 caused cross-fabric QP RTR timeouts. - disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 256 - cuda-graph-max-bs-decode: 256 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 8 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - - decode: - nodes: 1 - workers: 1 - gpus: 8 - - - env: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_RAGGED_VERIFY_MODE: "static" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_DSV4_MHC_PREWARM: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 - disaggregation-mode: decode - load-balance-method: total_tokens - # Leave enough decode activation headroom while expanding the KV pools - # for transient DP-rank imbalance at concurrency 256. - mem-fraction-static: 0.91 - page-size: 256 - # Favor the full-attention pool while retaining enough SWA capacity for - # the busiest decode rank. - swa-full-tokens-ratio: 0.005 - max-running-requests: 512 - cuda-graph-max-bs-decode: 256 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; - # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. - cpus-per-task: "192" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..9b729ab3fd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,263 @@ +# AgentX dsv4 sglang b200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" + frameworks: + dynamo: "1.5.0.dev20260914" + dynamo: + install: true + source: + wheel: "1.5.0.dev20260914" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: b200 + gpus_per_node: 8 + services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + # P/D workers are on separate HGX B200 nodes, so use Mooncake's standard + # RDMA-registered CUDA buffers rather than an NVL72 custom memory pool. + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + # Use the eight external HCAs. mlx5_6..9 are on a separate low-LID + # fabric, while mlx5_bond_0 caused cross-fabric QP RTR timeouts. + disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.85 + page-size: 256 + # Favor the full-attention pool; measured SWA utilization remained low. + swa-full-tokens-ratio: 0.01 + max-running-requests: 256 + cuda-graph-max-bs-decode: 256 + chunked-prefill-size: 65536 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 8 + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_RAGGED_VERIFY_MODE: "static" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + disaggregation-ib-device: mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + page-size: 256 + swa-full-tokens-ratio: 0.02 + cuda-graph-max-bs-decode: 256 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + sbatch_directives: + mem: "0" + # NScale B200 nodes expose 192 logical CPUs. Request the full node CPU set; + # mem=0 reserves all 1.7 TiB of allocatable host DRAM for HiCache. + cpus-per-task: "192" + srun_options: + mem: "0" + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + +# B200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 +# (1P x DEP8 / 1D x DEP8, bundled DSpark + HiCache KV offload), tuned for concurrency 64. +# Each DEP8 worker occupies one eight-GPU B200 node. +override_1p1d_c64: + name: "disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload" + roles: + decode: + args: + max-running-requests: 128 + +# B200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 +# (1P x DEP8 / 1D x DEP8, bundled DSpark + HiCache KV offload), tuned for concurrency 128. +# Each DEP8 worker occupies one eight-GPU B200 node. +override_1p1d_c128: + name: "disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload" + roles: + decode: + args: + max-running-requests: 256 + +# B200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 +# (2P x DEP8 / 1D x DEP8, bundled DSpark + HiCache KV offload), tuned for concurrency 256. +# Each DEP8 worker occupies one eight-GPU B200 node. +override_2p1d_c256: + name: "disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload" + roles: + prefill: + nodes: 2 + workers: 2 + args: + swa-full-tokens-ratio: 0.02 + decode: + env: + SGLANG_DSV4_MHC_PREWARM: '1' + args: + # Leave enough decode activation headroom while expanding the KV pools + # for transient DP-rank imbalance at concurrency 256. + mem-fraction-static: 0.91 + # Favor the full-attention pool while retaining enough SWA capacity for + # the busiest decode rank. + swa-full-tokens-ratio: 0.005 + max-running-requests: 512 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml deleted file mode 100644 index c49541b92b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml +++ /dev/null @@ -1,130 +0,0 @@ -schema: 2 -name: "agg-gb300-tp4-mtp-lowlatency" - -# Low-latency AgentX aggregate topology: one TP4 worker occupies one -# four-GPU GB300 node and serves both prefill and decode with DSpark K=6. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260910" - -slurm: - time_limit: "4:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: high - PIP_BREAK_SYSTEM_PACKAGES: "1" - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" - enable-metrics: true - enable-cache-report: true - trust-remote-code: true - weight-loader-prefetch-checkpoints: true - stream-interval: 10 - watchdog-timeout: 1000000 - mem-fraction-static: 0.94 - page-size: 256 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - moe-runner-backend: "flashinfer_mxfp4" - enable-deepseek-v4-fp4-indexer: true - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 32 - cuda-graph-max-bs-decode: 32 - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 4 - ep-size: 1 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "false" - TP: "4" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml deleted file mode 100644 index 0aea65fbef..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml +++ /dev/null @@ -1,130 +0,0 @@ -schema: 2 -name: "agg-gb300-tp8-mtp-lowlatency" - -# Low-latency AgentX aggregate topology: one TP8 worker spans two -# four-GPU GB300 nodes and serves both prefill and decode with DSpark K=6. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260910" - -slurm: - time_limit: "4:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: high - PIP_BREAK_SYSTEM_PACKAGES: "1" - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" - enable-metrics: true - enable-cache-report: true - trust-remote-code: true - weight-loader-prefetch-checkpoints: true - stream-interval: 10 - watchdog-timeout: 1000000 - mem-fraction-static: 0.94 - page-size: 256 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - moe-runner-backend: "flashinfer_mxfp4" - enable-deepseek-v4-fp4-indexer: true - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 4 - cuda-graph-max-bs-decode: 4 - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 8 - ep-size: 1 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "false" - TP: "8" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..2ef179373b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -0,0 +1,146 @@ +# AgentX dsv4 sglang gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9" + dynamo: + install: true + source: + wheel: "1.5.0.dev20260910" + slurm: + time_limit: "4:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: "gb300" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + agg: + workers: 1 + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" + enable-metrics: true + enable-cache-report: true + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.94 + page-size: 256 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + enable-deepseek-v4-fp4-indexer: true + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + scheduler-recv-interval: 30 + dp-size: 1 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + sbatch_directives: + mem: "0" + cpus-per-task: "144" + srun_options: + mem: "0" + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + +# Low-latency AgentX aggregate topology: one TP4 worker occupies one +# four-GPU GB300 node and serves both prefill and decode with DSpark K=6. +override_tp4: + name: "agg-gb300-tp4-mtp-lowlatency" + roles: + agg: + nodes: 1 + gpus: 4 + args: + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + tp-size: 4 + benchmark: + env: + TP: "4" + +# Low-latency AgentX aggregate topology: one TP8 worker spans two +# four-GPU GB300 nodes and serves both prefill and decode with DSpark K=6. +override_tp8: + name: "agg-gb300-tp8-mtp-lowlatency" + roles: + agg: + nodes: 2 + gpus: 8 + args: + max-running-requests: 4 + cuda-graph-max-bs-decode: 4 + tp-size: 8 + benchmark: + env: + TP: "8" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml deleted file mode 100644 index 15885fb9af..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml +++ /dev/null @@ -1,234 +0,0 @@ -schema: 2 -name: "disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 480. -# -# Uses the flat single-variant srtctl schema the agentic CI flow expects; -# resources + backend (prefill/decode env + sglang_config) are normalized -# from the Pareto run. -# Concurrency is exported into agentic_srt.sh from the master-config conc-list. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260902" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 256 - cuda-graph-max-bs: 256 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 16 - dp-size: 16 - ep-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 3072 - cuda-graph-max-bs: 256 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml deleted file mode 100644 index 00ce318b0d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml +++ /dev/null @@ -1,236 +0,0 @@ -schema: 2 -name: "disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 -# (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 960. -# -# DEP8-prefill variant aligned with the measured Pareto point: prefill uses -# tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. -# Concurrency is exported into agentic_srt.sh -# from the master-config conc-list. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260902" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - prefill: - nodes: 4 - workers: 2 - env: - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - OMP_NUM_THREADS: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_DSV4_MHC_PREWARM: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 256 - cuda-graph-max-bs: 256 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - nodes: 4 - workers: 1 - - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - OMP_NUM_THREADS: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_DSV4_MHC_PREWARM: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 16 - dp-size: 16 - ep-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 3072 - cuda-graph-max-bs: 256 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml deleted file mode 100644 index f0e66a32d8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml +++ /dev/null @@ -1,234 +0,0 @@ -schema: 2 -name: "disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1440. -# -# Uses the flat single-variant srtctl schema the agentic CI flow expects; -# resources + backend (prefill/decode env + sglang_config) are normalized -# from the Pareto run. -# Concurrency is exported into agentic_srt.sh from the master-config conc-list. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260902" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - prefill: - nodes: 6 - workers: 3 - gpus: 8 - env: - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 16 - dp-size: 16 - ep-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 3072 - cuda-graph-max-bs: 512 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml deleted file mode 100644 index e569bd41cd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml +++ /dev/null @@ -1,239 +0,0 @@ -schema: 2 -name: "disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1920. -# -# Uses the flat single-variant srtctl schema the agentic CI flow expects; -# resources + backend (prefill/decode env + sglang_config) are normalized -# from the Pareto run. -# Concurrency is exported into agentic_srt.sh from the master-config conc-list. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" - -dynamo: - install: true - source: - wheel: "1.5.0.dev20260902" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - nginx_keepalive_timeout: "900s" - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - # AgentX warmup can legitimately keep the single wide decode worker busy - # for longer than Dynamo's 10-second TCP request-plane default. - DYN_TCP_REQUEST_TIMEOUT: "60" - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -engine: sglang -roles: - prefill: - nodes: 8 - workers: 4 - gpus: 8 - env: - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - - args: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 16 - dp-size: 16 - ep-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 3072 - cuda-graph-max-bs: 192 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..ad2af4be37 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,313 @@ +# AgentX dsv4 sglang gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" + dynamo: + install: true + source: + wheel: "1.5.0.dev20260902" + slurm: + time_limit: "8:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + prefill: + env: + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.85 + page-size: 256 + swa-full-tokens-ratio: 0.02 + chunked-prefill-size: 65536 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + decode: + nodes: 4 + workers: 1 + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 3072 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + sbatch_directives: + mem: "0" + cpus-per-task: "144" + srun_options: + mem: "0" + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 480. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into srt_agentic.sh from the master-config conc-list. +override_1p1d_c480: + name: "disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload" + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + args: + max-running-requests: 256 + cuda-graph-max-bs: 256 + decode: + gpus: 16 + args: + cuda-graph-max-bs: 256 + +# Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 +# (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 960. +# +# DEP8-prefill variant aligned with the measured Pareto point: prefill uses +# tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. +# Concurrency is exported into srt_agentic.sh +# from the master-config conc-list. +override_2p1d_c960: + name: "disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload" + roles: + prefill: + nodes: 4 + workers: 2 + env: + OMP_NUM_THREADS: '1' + args: + max-running-requests: 256 + cuda-graph-max-bs: 256 + decode: + env: + OMP_NUM_THREADS: '1' + SGLANG_DSV4_MHC_PREWARM: '1' + args: + cuda-graph-max-bs: 256 + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1440. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into srt_agentic.sh from the master-config conc-list. +override_3p1d_c1440: + name: "disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload" + roles: + prefill: + nodes: 6 + workers: 3 + gpus: 8 + args: + max-running-requests: 512 + cuda-graph-max-bs: 512 + decode: + gpus: 16 + args: + cuda-graph-max-bs: 512 + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1920. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into srt_agentic.sh from the master-config conc-list. +override_4p1d_c1920: + name: "disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload" + frontend: + nginx_keepalive_timeout: "900s" + env: + # AgentX warmup can legitimately keep the single wide decode worker busy + # for longer than Dynamo's 10-second TCP request-plane default. + DYN_TCP_REQUEST_TIMEOUT: "60" + roles: + prefill: + nodes: 8 + workers: 4 + gpus: 8 + env: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + args: + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + decode: + gpus: 16 + env: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + args: + cuda-graph-max-bs: 192 + benchmark: + env: + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml index d1b42f5e6b..d3c670c0f0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml @@ -127,7 +127,7 @@ benchmark: type: custom # Replaced with CONC_LIST by the launcher before submission. concurrencies: [1] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml new file mode 100644 index 0000000000..05ac40f3de --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml @@ -0,0 +1,726 @@ +# DeepSeek-V4-Pro-0813 AgentX on MI355X: 1P1D SGLang disaggregation over MoRI +# with DSpark. Every arm offloads prefill KV to UMBP as a direct external store +# for the unified radix tree (no host cache tier), backed by a 1.5 TB +# hugepage DRAM tier on the prefill node. +base: + name: mi355x-dsv4-pro-0813-agentx-umbp + model: + path: DeepSeek-V4-Pro-0813 + container: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260923 + precision: fp4 + slurm: + time_limit: "08:00:00" + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang-router + enable_multiple_frontends: false + args: + policy: consistent_hashing + dp-aware: true + cache-threshold: 0.3 + balance-abs-threshold: 2 + balance-rel-threshold: 1.1 + disable-circuit-breaker: true + health-failure-threshold: 100 + health-check-timeout-secs: 600 + health-check-interval-secs: 30 + engine: sglang + # A recipe host_setup replaces the cluster default, so run its fabric, GPU and + # hugepage-reclaim checks first. Then reserve the UMBP tier's hugepages on the + # prefill node (the allocation's first node in 1P1D) so the tier never falls + # back to 4 KiB pages; teardown returns them for the next job. + host_setup: + nodes: workers + timeout_seconds: 1800 + commands: + - IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" + - >- + if [[ "$(hostname -s)" == "$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)" ]]; then + page_kb=$(awk '/^Hugepagesize:/ {print $2}' /proc/meminfo); + pages=$(( (1500000000000 / (page_kb * 1024) + 1) * 102 / 100 )); + HUGEPAGES_TARGET=$pages HUGEPAGE_GROW_TOLERANCE_PCT=1 + bash "${GITHUB_WORKSPACE}/benchmarks/multi_node/amd_utils/helpers/hugepage_sanity.sh"; fi + teardown: + - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/benchmarks/multi_node/amd_utils/helpers/hugepage_sanity.sh" + # The per-allocation /logs mount is shared by the service and prefill + # containers, so the socket cannot collide with another job's UMBP. + services: + - name: umbp + type: generic + placement: + node: prefill + start: before_workers + critical: true + inherit_discovery_env: false + command: + - /sgl-workspace/mori/python/mori/umbp_standalone_server + - unix:///logs/umbp.sock + env: + HSA_NO_SCRATCH_RECLAIM: "0" + GPU_MAX_HW_QUEUES: "5" + UMBP_DRAM_CAPACITY: "1500000000000" + UMBP_DRAM_USE_HUGEPAGES: "1" + UMBP_SSD_ENABLED: "0" + MORI_UMBP_LOG_LEVEL: info + preamble: | + host_gb=$(awk '/^MemTotal:/ {printf "%d", $2 / 1000000}' /proc/meminfo) + if (( UMBP_DRAM_CAPACITY / 1000000000 > host_gb / 2 )); then + echo "UMBP capacity exceeds half of host memory" >&2 + exit 1 + fi + export LD_LIBRARY_PATH="/sgl-workspace/mori/python/mori:${LD_LIBRARY_PATH}" + readiness: + # Wait for GPU registration of the tier, not merely a listening socket. + log: + pattern: host memory registered for GPU access + timeout_seconds: 5400 + interval_seconds: 1 + roles: + prefill: + nodes: 1 + workers: 1 + env: &environment + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub + PYTHONDONTWRITEBYTECODE: "1" + PYTHONUNBUFFERED: "1" + PYTHONPATH: /sgl-workspace/aiter + TORCH_NCCL_BLOCKING_WAIT: "1" + NCCL_BLOCKING_WAIT: "1" + NCCL_IB_HCA: rdma3,rdma0,rdma2,rdma1,rdma7,rdma4,rdma6,rdma5 + GLOO_SOCKET_IFNAME: eno0 + NCCL_SOCKET_IFNAME: eno0 + AITER_LOG_LEVEL: ERROR + HSA_NO_SCRATCH_RECLAIM: "0" + GPU_MAX_HW_QUEUES: "2" + SGLANG_AITER_MLA_PERSIST: "0" + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_MORI_DISPATCH_DTYPE: auto + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "4096" + SGLANG_MORI_RECV_BOUND: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_USE_AITER: "1" + SGLANG_USE_ROCM700A: "0" + SGLANG_HACK_FLASHMLA_BACKEND: unified_kv_triton + SGLANG_OPT_USE_AITER_INDEXER: "true" + SGLANG_OPT_FP8_WO_A_FUSED_INVROPE: "1" + SGLANG_OPT_USE_AITER_BATCHED_GEMM: "1" + AITER_BF16_FP8_MOE_BOUND: "0" + TORCH_BLAS_PREFER_HIPBLASLT: "1" + IBDEVICES: rdma3,rdma0,rdma2,rdma1,rdma7,rdma4,rdma6,rdma5 + MORI_RDMA_TC: "104" + MORI_IO_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "500000" + MORI_IO_QP_MAX_SEND_WR: "32767" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_IO_TC_DISABLE: "0" + MORI_SHMEM_MODE: ISOLATION + MORI_EP_LAUNCH_CONFIG_MODE: AUTO + MORI_APP_LOG_LEVEL: WARNING + SGLANG_MORI_COMBINE_DTYPE: auto + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + MORI_MAX_DISPATCH_TOKENS_PREFILL: "8192" + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_ENABLE_OVERLAP_PLAN_STREAM: "0" + SGLANG_LOG_MS: "true" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_ROUTER_STDOUT_LOGS: "0" + UMBP_STANDALONE_ADDRESS: unix:///logs/umbp.sock + args: &server + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tp-size: 8 + enable-deepseek-v4-fp4-indexer: true + watchdog-timeout: 3600 + load-balance-method: round_robin + kv-cache-dtype: fp8_e4m3 + attention-backend: dsv4 + page-size: 256 + swa-full-tokens-ratio: 0.1 + enforce-shared-experts-fusion: true + tool-call-parser: deepseekv4 + reasoning-parser: deepseek-v4 + disaggregation-transfer-backend: mori + disaggregation-ib-device: rdma3,rdma0,rdma2,rdma1,rdma7,rdma4,rdma6,rdma5 + tokenizer-worker-num: 8 + stream-interval: 20 + log-level: info + log-level-http: error + speculative-algorithm: DSPARK + speculative-eagle-topk: 1 + speculative-num-steps: 1 + context-length: 1048576 + enable-metrics: true + disable-cuda-graph: true + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mori + hicache-storage-backend-extra-config: '{"standalone_startup_timeout_ms":120000}' + enable-cache-report: true + # Optimistic prefill (sgl-project/sglang#38978) is wired for the linker only. + optimistic-prefill-attempts: 2 + decode: + nodes: 1 + workers: 1 + env: + <<: *environment + # Decode runs no UMBP tier; offload is prefill-side. + UMBP_STANDALONE_ADDRESS: "" + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "128" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + <<: *server + disable-cuda-graph: false + sbatch_directives: + cpus-per-task: "128" + mem: "0" + srun_options: + mem: "0" + container-writable: "" + container-remap-root: "" + health_check: + max_attempts: 720 + interval_seconds: 5 + benchmark: + type: custom + container_image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907 + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + # The MI355X launcher collects results from the job workspace. + RESULT_DIR: /infmax-workspace/LOGS/agentic + AGENTIC_OUTPUT_DIR: /infmax-workspace + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache/hub + TOKENIZERS_PARALLELISM: "false" + TRANSFORMERS_VERBOSITY: error + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + +# One variant per point; each point runs in its own allocation. Admission is +# 2x CONC on both roles. Decode captures graphs up to 2x CONC, or CONC/4 per +# rank under DP attention. DSpark draft length is 6 at c4/c16 (the golden AL +# peak) and 3 above; MoRI decode dispatch scales with it (64 x (draft + 1)). +override_c4: + roles: + prefill: + gpus: 4 + args: + tp-size: 4 + mem-fraction-static: 0.86 + chunked-prefill-size: 16384 + max-running-requests: 8 + speculative-dspark-block-size: 6 + speculative-num-draft-tokens: 7 + decode: + gpus: 4 + env: + MORI_MAX_DISPATCH_TOKENS_DECODE: "448" + args: + tp-size: 4 + mem-fraction-static: 0.86 + max-running-requests: 8 + cuda-graph-bs-decode: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + speculative-dspark-block-size: 6 + speculative-num-draft-tokens: 7 + benchmark: + env: + CONC: '4' + +override_c16: + roles: + prefill: + gpus: 4 + args: + tp-size: 4 + mem-fraction-static: 0.86 + chunked-prefill-size: 16384 + max-running-requests: 32 + speculative-dspark-block-size: 6 + speculative-num-draft-tokens: 7 + decode: + gpus: 8 + env: + MORI_MAX_DISPATCH_TOKENS_DECODE: "448" + args: + tp-size: 8 + mem-fraction-static: 0.86 + max-running-requests: 32 + cuda-graph-bs-decode: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 + - 22 + - 23 + - 24 + - 25 + - 26 + - 27 + - 28 + - 29 + - 30 + - 31 + - 32 + speculative-dspark-block-size: 6 + speculative-num-draft-tokens: 7 + benchmark: + env: + CONC: '16' + +override_c32: + roles: + prefill: + gpus: 4 + args: + tp-size: 4 + mem-fraction-static: 0.86 + chunked-prefill-size: 16384 + max-running-requests: 64 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + decode: + gpus: 8 + env: + MORI_MAX_DISPATCH_TOKENS_DECODE: "256" + args: + tp-size: 8 + mem-fraction-static: 0.86 + max-running-requests: 64 + cuda-graph-bs-decode: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 + - 22 + - 23 + - 24 + - 25 + - 26 + - 27 + - 28 + - 29 + - 30 + - 31 + - 32 + - 33 + - 34 + - 35 + - 36 + - 37 + - 38 + - 39 + - 40 + - 41 + - 42 + - 43 + - 44 + - 45 + - 46 + - 47 + - 48 + - 49 + - 50 + - 51 + - 52 + - 53 + - 54 + - 55 + - 56 + - 57 + - 58 + - 59 + - 60 + - 61 + - 62 + - 63 + - 64 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + benchmark: + env: + CONC: '32' + +override_c48: + roles: + prefill: + gpus: 4 + args: + tp-size: 4 + mem-fraction-static: 0.86 + chunked-prefill-size: 16384 + max-running-requests: 96 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + decode: + gpus: 8 + env: + MORI_MAX_DISPATCH_TOKENS_DECODE: "256" + args: + tp-size: 8 + mem-fraction-static: 0.86 + max-running-requests: 96 + cuda-graph-bs-decode: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 + - 22 + - 23 + - 24 + - 25 + - 26 + - 27 + - 28 + - 29 + - 30 + - 31 + - 32 + - 33 + - 34 + - 35 + - 36 + - 37 + - 38 + - 39 + - 40 + - 41 + - 42 + - 43 + - 44 + - 45 + - 46 + - 47 + - 48 + - 49 + - 50 + - 51 + - 52 + - 53 + - 54 + - 55 + - 56 + - 57 + - 58 + - 59 + - 60 + - 61 + - 62 + - 63 + - 64 + - 65 + - 66 + - 67 + - 68 + - 69 + - 70 + - 71 + - 72 + - 73 + - 74 + - 75 + - 76 + - 77 + - 78 + - 79 + - 80 + - 81 + - 82 + - 83 + - 84 + - 85 + - 86 + - 87 + - 88 + - 89 + - 90 + - 91 + - 92 + - 93 + - 94 + - 95 + - 96 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + benchmark: + env: + CONC: '48' + +override_c192: + roles: + prefill: + gpus: 8 + env: + GPU_MAX_HW_QUEUES: "5" + SGLANG_SHARED_EXPERT_TP1: "1" + SGLANG_DP_SHARED_EXPERT_LOCAL: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_DP_USE_REDUCE_SCATTER: "1" + args: + tp-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + swa-full-tokens-ratio: 0.15 + mem-fraction-static: 0.92 + chunked-prefill-size: 65536 + enable-two-batch-overlap: false + max-running-requests: 384 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + decode: + gpus: 8 + env: + GPU_MAX_HW_QUEUES: "5" + SGLANG_SHARED_EXPERT_TP1: "1" + SGLANG_DP_SHARED_EXPERT_LOCAL: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_DP_USE_REDUCE_SCATTER: "1" + MORI_MAX_DISPATCH_TOKENS_DECODE: "256" + args: + tp-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + swa-full-tokens-ratio: 0.15 + mem-fraction-static: 0.92 + max-running-requests: 384 + cuda-graph-bs-decode: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 + - 22 + - 23 + - 24 + - 25 + - 26 + - 27 + - 28 + - 29 + - 30 + - 31 + - 32 + - 33 + - 34 + - 35 + - 36 + - 37 + - 38 + - 39 + - 40 + - 41 + - 42 + - 43 + - 44 + - 45 + - 46 + - 47 + - 48 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + benchmark: + env: + CONC: '192' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + +override_c256: + roles: + prefill: + gpus: 8 + env: + GPU_MAX_HW_QUEUES: "5" + SGLANG_SHARED_EXPERT_TP1: "1" + SGLANG_DP_SHARED_EXPERT_LOCAL: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_DP_USE_REDUCE_SCATTER: "1" + args: + tp-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + swa-full-tokens-ratio: 0.15 + mem-fraction-static: 0.92 + chunked-prefill-size: 65536 + enable-two-batch-overlap: false + max-running-requests: 512 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + decode: + gpus: 8 + env: + GPU_MAX_HW_QUEUES: "5" + SGLANG_SHARED_EXPERT_TP1: "1" + SGLANG_DP_SHARED_EXPERT_LOCAL: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_DP_USE_REDUCE_SCATTER: "1" + MORI_MAX_DISPATCH_TOKENS_DECODE: "256" + args: + tp-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + swa-full-tokens-ratio: 0.15 + mem-fraction-static: 0.92 + max-running-requests: 512 + cuda-graph-bs-decode: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 + - 22 + - 23 + - 24 + - 25 + - 26 + - 27 + - 28 + - 29 + - 30 + - 31 + - 32 + - 33 + - 34 + - 35 + - 36 + - 37 + - 38 + - 39 + - 40 + - 41 + - 42 + - 43 + - 44 + - 45 + - 46 + - 47 + - 48 + - 49 + - 50 + - 51 + - 52 + - 53 + - 54 + - 55 + - 56 + - 57 + - 58 + - 59 + - 60 + - 61 + - 62 + - 63 + - 64 + speculative-dspark-block-size: 3 + speculative-num-draft-tokens: 4 + benchmark: + env: + CONC: '256' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml deleted file mode 100644 index adc2d1d3a3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml +++ /dev/null @@ -1,195 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 256 - max_num_tokens: 16384 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 8 - decode: - nodes: 8 - workers: 1 - gpus: 32 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.7 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 4 - max_num_tokens: 16 - max_seq_len: 996595 - moe_config: - backend: MEGAMOE_DEEPGEMM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 32 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '388' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: first_decode -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '388' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro - placement: - node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml deleted file mode 100644 index f4f16337f4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml +++ /dev/null @@ -1,195 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 128 - max_num_tokens: 4096 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 4 - decode: - nodes: 8 - workers: 4 - gpus: 8 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.9 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 1 - max_num_tokens: 4 - max_seq_len: 996595 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 8 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '4' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: first_decode -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '4' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro - placement: - node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml deleted file mode 100644 index 08b68b2202..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml +++ /dev/null @@ -1,195 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 128 - max_num_tokens: 4096 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 4 - decode: - nodes: 6 - workers: 6 - gpus: 4 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.9 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 4 - max_num_tokens: 16 - max_seq_len: 996595 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '24' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: first_decode -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '24' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro - placement: - node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml deleted file mode 100644 index 46be52c30f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml +++ /dev/null @@ -1,196 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 4 - workers: 2 - gpus: 8 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 256 - max_num_tokens: 16384 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 8 - decode: - nodes: 8 - workers: 1 - gpus: 32 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.7 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 8 - max_num_tokens: 32 - max_seq_len: 996595 - moe_config: - backend: MEGAMOE_DEEPGEMM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 32 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '736' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: first_decode -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '736' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro - placement: - node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml deleted file mode 100644 index 373a12989c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml +++ /dev/null @@ -1,202 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 6 - workers: 3 - gpus: 8 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 256 - max_num_tokens: 16384 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 8 - decode: - nodes: 4 - workers: 1 - gpus: 16 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 12 - - 16 - - 20 - - 24 - - 28 - - 32 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.7 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 32 - max_num_tokens: 128 - max_seq_len: 996595 - moe_config: - backend: MEGAMOE_DEEPGEMM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 16 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '1152' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: first_decode -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '1152' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro - placement: - node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml deleted file mode 100644 index 6e2e980ebd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml +++ /dev/null @@ -1,218 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 10 - workers: 5 - gpus: 8 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 256 - max_num_tokens: 16384 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 8 - decode: - nodes: 4 - workers: 1 - gpus: 16 - env: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 12 - - 16 - - 20 - - 24 - - 28 - - 32 - - 36 - - 40 - - 44 - - 48 - - 52 - - 56 - - 60 - - 64 - - 68 - - 72 - - 76 - - 80 - - 84 - - 88 - - 92 - - 96 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.7 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 96 - max_num_tokens: 384 - max_seq_len: 996595 - moe_config: - backend: MEGAMOE_DEEPGEMM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 16 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '2626' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: first_decode -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '2626' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro - placement: - node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..d78570c07f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,399 @@ +# AgentX dsv4 trtllm gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + identity: + model: + repo: deepseek-ai/DeepSeek-V4-Pro + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + request_plane: tcp + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + engine: + type: trtllm + publish_events_and_metrics: false + roles: + prefill: + gpus: 8 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + CUDA_SCALE_LAUNCH_QUEUES: 4x + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 193273528320 + pool_ratio: + - 0.55 + - 0.22 + - 0.23 + tokens_per_block: 128 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + max_batch_size: 256 + max_num_tokens: 16384 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 8 + decode: + workers: 1 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.7 + host_cache_size: 0 + tokens_per_block: 128 + max_seq_len: 996595 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + frontend: + type: dynamo + enable_multiple_frontends: false + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + DURATION: '3600' + KV_OFFLOADING: dram + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: first_decode + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + DURATION: '3600' + KV_OFFLOADING: dram + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: DeepSeek-V4-Pro + placement: + node: last_decode + +override_1p4d_dep4_tep8_c4_b1: + name: dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + args: + max_batch_size: 128 + max_num_tokens: 4096 + moe_expert_parallel_size: 4 + tensor_parallel_size: 4 + decode: + nodes: 8 + workers: 4 + gpus: 8 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + free_gpu_memory_fraction: 0.9 + max_batch_size: 1 + max_num_tokens: 4 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + tensor_parallel_size: 8 + frontend: + env: + CONC: '4' + benchmark: + env: + CONC: '4' + +override_1p6d_dep4_tep4_c24_b4: + name: dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + args: + max_batch_size: 128 + max_num_tokens: 4096 + moe_expert_parallel_size: 4 + tensor_parallel_size: 4 + decode: + nodes: 6 + workers: 6 + gpus: 4 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + free_gpu_memory_fraction: 0.9 + max_batch_size: 4 + max_num_tokens: 16 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + tensor_parallel_size: 4 + frontend: + env: + CONC: '24' + benchmark: + env: + CONC: '24' + +override_1p1d_dep8_dep32_c388_b4: + name: dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp + roles: + prefill: + nodes: 2 + workers: 1 + decode: + nodes: 8 + gpus: 32 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + max_batch_size: 4 + max_num_tokens: 16 + moe_expert_parallel_size: 32 + tensor_parallel_size: 32 + frontend: + env: + CONC: '388' + benchmark: + env: + CONC: '388' + +override_2p1d_dep8_dep32_c736_b8: + name: dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp + roles: + prefill: + nodes: 4 + workers: 2 + decode: + nodes: 8 + gpus: 32 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + max_batch_size: 8 + max_num_tokens: 32 + moe_expert_parallel_size: 32 + tensor_parallel_size: 32 + frontend: + env: + CONC: '736' + benchmark: + env: + CONC: '736' + +override_3p1d_dep8_dep16_c1152_b32: + name: dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp + roles: + prefill: + nodes: 6 + workers: 3 + decode: + nodes: 4 + gpus: 16 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + max_batch_size: 32 + max_num_tokens: 128 + moe_expert_parallel_size: 16 + tensor_parallel_size: 16 + frontend: + env: + CONC: '1152' + benchmark: + env: + CONC: '1152' + +override_5p1d_dep8_dep16_c2626_b96: + name: dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp + roles: + prefill: + nodes: 10 + workers: 5 + decode: + nodes: 4 + gpus: 16 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + - 36 + - 40 + - 44 + - 48 + - 52 + - 56 + - 60 + - 64 + - 68 + - 72 + - 76 + - 80 + - 84 + - 88 + - 92 + - 96 + max_batch_size: 96 + max_num_tokens: 384 + moe_expert_parallel_size: 16 + tensor_parallel_size: 16 + frontend: + env: + CONC: '2626' + benchmark: + env: + CONC: '2626' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml deleted file mode 100644 index 9c604d726b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml +++ /dev/null @@ -1,112 +0,0 @@ -schema: 2 -name: dsv4-gb200-vllm-agentic-mtp-agg-dep8 - -model: - path: deepseek-v4-pro - container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - precision: fp4 - -identity: - model: {repo: deepseek-ai/DeepSeek-V4-Pro} - container: {image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106} - frameworks: {dynamo: "1.3.1"} - -dynamo: - install: true - - source: - pypi: "1.3.1" -setup_script: vllm-container-deps.sh -environment: {ETCD_LEASE_TTL: "7200"} -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: gb200 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: kv - router-reset-states: true - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 256 - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: /hf_hub_cache - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - args: - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 8192 - trust-remote-code: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - gpu-memory-utilization: 0.90 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - kv-cache-metrics: true - tokenizer-mode: deepseek_v4 - -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml new file mode 100644 index 0000000000..87fa054566 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml @@ -0,0 +1,135 @@ +# AgentX dsv4 vllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml:override_. + +schema: 2 + +base: + model: + path: deepseek-v4-pro + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + precision: fp4 + identity: + model: + repo: deepseek-ai/DeepSeek-V4-Pro + container: + image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + frameworks: + dynamo: "1.3.1" + dynamo: + install: true + source: + pypi: "1.3.1" + setup_script: vllm-container-deps.sh + environment: + ETCD_LEASE_TTL: "7200" + slurm: + time_limit: "8:00:00" + health_check: + max_attempts: 2160 + interval_seconds: 10 + resources: + gpu_type: gb200 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: kv + router-reset-states: true + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 256 + engine: + type: vllm + connector: + roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + pipeline-parallel-size: 1 + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-batched-tokens: 8192 + trust-remote-code: true + block-size: 256 + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + gpu-memory-utilization: 0.90 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + kv-cache-metrics: true + tokenizer-mode: deepseek_v4 + sbatch_directives: + cpus-per-task: "144" + mem: "0" + srun_options: + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + +override_tp8: + name: dsv4-gb200-vllm-agentic-mtp-agg-tp8 + roles: + agg: + args: + tensor-parallel-size: 8 + disable-custom-all-reduce: true + max-num-seqs: 128 + no-enable-flashinfer-autotune: true + +override_dep8: + name: dsv4-gb200-vllm-agentic-mtp-agg-dep8 + engine: + dp_launch_mode: per_node + roles: + agg: + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + max-num-seqs: 256 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml new file mode 100644 index 0000000000..bf9c7321ba --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml @@ -0,0 +1,176 @@ +# AgentX dsv4 vllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.2.1" + dynamo: + install: true + source: + wheel: "1.2.1" + environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + # The frontend shares Grace CPU capacity with the long TP8 cold start. + ETCD_LEASE_TTL: "7200" + setup_script: vllm-container-deps.sh + slurm: + time_limit: "8:00:00" + health_check: + max_attempts: 2160 + interval_seconds: 10 + resources: + gpu_type: "gb200" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + engine: + type: vllm + connector: + roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + gpu-memory-utilization: 0.94 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + sbatch_directives: + cpus-per-task: "144" + mem: "0" + srun_options: + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep aggregate workers in the multinode result schema so ingestion uses + # the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +# GB200 AgentX aggregate topology: one TP8 worker spans two four-GPU +# nodes and serves both prefill and decode at concurrency 4. Keep at least +# 16 sequence slots and otherwise size the scheduler at 4x concurrency. +override_c4: + name: "svf-vllm-agg-gb200-tp8-c4-mtp2-agentic" + roles: + agg: + env: + VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-agg-mtp2-{job_id}" + args: + max-num-seqs: 16 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' + +# GB200 AgentX aggregate topology: one TP8 worker spans two four-GPU +# nodes and serves both prefill and decode at concurrency 8. Size max-num-seqs at +# 4x concurrency and expand the MTP CUDA-graph envelope to match. +override_c8: + name: "svf-vllm-agg-gb200-tp8-c8-mtp2-agentic" + roles: + agg: + env: + VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-agg-tp8-c8-mtp2-{job_id}" + args: + max-num-seqs: 32 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' + benchmark: + env: + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml deleted file mode 100644 index 71f279e59b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml +++ /dev/null @@ -1,160 +0,0 @@ -schema: 2 -name: "svf-vllm-agg-gb200-tp8-c4-mtp2-agentic" - -# GB200 AgentX aggregate topology: one TP8 worker spans two four-GPU -# nodes and serves both prefill and decode at concurrency 4. Keep at least -# 16 sequence slots and otherwise size the scheduler at 4x concurrency. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - install: true - - source: - wheel: "1.2.1" -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - # The frontend shares Grace CPU capacity with the long TP8 cold start. - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -engine: - type: vllm - connector: -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-agg-mtp2-{job_id}" - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 16 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep aggregate workers in the multinode result schema so ingestion uses - # the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml deleted file mode 100644 index 58e999b60d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml +++ /dev/null @@ -1,159 +0,0 @@ -schema: 2 -name: "svf-vllm-agg-gb200-tp8-c8-mtp2-agentic" - -# GB200 AgentX aggregate topology: one TP8 worker spans two four-GPU -# nodes and serves both prefill and decode at concurrency 8. Size max-num-seqs at -# 4x concurrency and expand the MTP CUDA-graph envelope to match. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - install: true - - source: - wheel: "1.2.1" -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -engine: - type: vllm - connector: -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-agg-tp8-c8-mtp2-{job_id}" - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml deleted file mode 100644 index 845cbf966f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml +++ /dev/null @@ -1,122 +0,0 @@ -schema: 2 -name: dsv4-gb200-vllm-agentic-mtp-agg-tp8 - -model: - path: deepseek-v4-pro - container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - precision: fp4 - -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - frameworks: - dynamo: "1.3.1" - -dynamo: - install: true - - source: - pypi: "1.3.1" -setup_script: vllm-container-deps.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: gb200 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: kv - router-reset-states: true - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 256 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: /hf_hub_cache - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - args: - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 128 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - gpu-memory-utilization: 0.90 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - kv-cache-metrics: true - tokenizer-mode: deepseek_v4 - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml deleted file mode 100644 index f9c62a6a24..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml +++ /dev/null @@ -1,221 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-c128-mtp2-agentic" - -# GB200 AgentX MTP3 topology: one DEP8 prefill worker feeds one -# DEP8 decode worker at concurrency 128. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - install: true - - source: - wheel: "1.3.0.dev20260720" -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "140GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c128-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c128-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 128 - max-num-batched-tokens: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 512 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml deleted file mode 100644 index 01c6e6e5af..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml +++ /dev/null @@ -1,221 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-c256-mtp2-agentic" - -# GB200 AgentX MTP3 topology: one DEP8 prefill worker feeds one -# DEP8 decode worker at concurrency 256. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - install: true - - source: - wheel: "1.3.0.dev20260720" -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "140GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 128 - max-num-batched-tokens: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 512 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml deleted file mode 100644 index 178f6af471..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml +++ /dev/null @@ -1,133 +0,0 @@ -schema: 2 -name: dsv4-gb200-vllm-agentic-mtp-disagg-1p1d-dep8-dep8 - -model: - path: deepseek-v4-pro - container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - precision: fp4 - -identity: - model: {repo: deepseek-ai/DeepSeek-V4-Pro} - container: {image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106} - frameworks: {dynamo: "1.3.1"} - -dynamo: - install: true - - source: - pypi: "1.3.1" -setup_script: vllm-container-deps.sh -environment: {ETCD_LEASE_TTL: "7200"} -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: gb200 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - # Long AgentX prefills can exceed Dynamo's request-plane default while - # the healthy DEP8 worker is still computing the first response. - DYN_TCP_REQUEST_TIMEOUT: "60" - args: - router-mode: kv - router-reset-states: true - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 256 - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: &worker_environment - HF_HUB_CACHE: /hf_hub_cache - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - args: &dep8_config - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - enforce-eager: true - block-size: 256 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - kv-cache-metrics: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: *worker_environment - args: - <<: *dep8_config - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' - enforce-eager: false - gpu-memory-utilization: 0.90 - max-num-seqs: 1024 - max-num-batched-tokens: 1024 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 - stream-interval: 10 - -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml deleted file mode 100644 index 0bc695d779..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml +++ /dev/null @@ -1,223 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb200-2p1d-dep8-dep12-c576-mtp2-agentic" - -# GB200 AgentX MTP3 topology: two DEP8 prefill workers feed one -# DEP12 decode worker at concurrency 576. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - install: true - - source: - wheel: "1.3.0.dev20260720" -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "140GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 4 - workers: 2 - gpus: 8 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-prefill-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 3 - workers: 1 - gpus: 12 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-dep12-decode-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 12 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml deleted file mode 100644 index 9ead5965b0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml +++ /dev/null @@ -1,223 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb200-2p1d-dep8-dep16-c512-mtp2-agentic" - -# GB200 AgentX MTP3 topology: two DEP8 prefill workers feed one -# DEP16 decode worker at concurrency 512. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - install: true - - source: - wheel: "1.3.0.dev20260720" -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "140GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 4 - workers: 2 - gpus: 8 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-prefill-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-dep16-decode-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml deleted file mode 100644 index fb540cdbf5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml +++ /dev/null @@ -1,130 +0,0 @@ -schema: 2 -name: dsv4-gb200-vllm-agentic-mtp-disagg-2p1d-dep8-dep8 - -model: - path: deepseek-v4-pro - container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - precision: fp4 - -identity: - model: {repo: deepseek-ai/DeepSeek-V4-Pro} - container: {image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106} - frameworks: {dynamo: "1.3.1"} - -dynamo: {install: true, source: {pypi: "1.3.1"}} -setup_script: vllm-container-deps.sh -environment: {ETCD_LEASE_TTL: "7200"} -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: gb200 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - # Long AgentX prefills can exceed Dynamo's request-plane default while - # the healthy DEP8 workers are still computing their first responses. - DYN_TCP_REQUEST_TIMEOUT: "60" - args: - router-mode: kv - router-reset-states: true - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 256 - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 4 - workers: 2 - gpus: 8 - env: &worker_environment - HF_HUB_CACHE: /hf_hub_cache - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - args: &dep8_config - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - enforce-eager: true - block-size: 256 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - kv-cache-metrics: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: *worker_environment - args: - <<: *dep8_config - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' - enforce-eager: false - gpu-memory-utilization: 0.90 - max-num-seqs: 1024 - max-num-batched-tokens: 1024 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 - stream-interval: 10 - -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml new file mode 100644 index 0000000000..57a3d6a417 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml @@ -0,0 +1,175 @@ +# AgentX dsv4 vllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml:override_. + +schema: 2 + +base: + model: + path: deepseek-v4-pro + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + precision: fp4 + identity: + model: {repo: deepseek-ai/DeepSeek-V4-Pro} + container: {image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106} + frameworks: {dynamo: "1.3.1"} + dynamo: + install: true + source: + pypi: "1.3.1" + setup_script: vllm-container-deps.sh + environment: {ETCD_LEASE_TTL: "7200"} + slurm: {time_limit: "8:00:00"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: gb200 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_TCP_REQUEST_TIMEOUT: "60" + args: + router-mode: kv + router-reset-states: true + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 256 + engine: + type: vllm + connector: + dp_launch_mode: per_node + roles: + prefill: + gpus: 8 + env: + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + enforce-eager: true + block-size: 256 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + kv-cache-metrics: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' + enforce-eager: false + gpu-memory-utilization: 0.90 + max-num-seqs: 1024 + max-num-batched-tokens: 1024 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + stream-interval: 10 + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + max-model-len: 1048576 + trust-remote-code: true + block-size: 256 + no-disable-hybrid-kv-cache-manager: true + kv-cache-metrics: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + sbatch_directives: {cpus-per-task: "144", mem: "0"} + srun_options: {container-remap-root: ""} + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + +# Long AgentX prefills can exceed Dynamo's request-plane default while +# the healthy DEP8 worker is still computing the first response. +override_1p1d: + name: dsv4-gb200-vllm-agentic-mtp-disagg-1p1d-dep8-dep8 + roles: + prefill: + nodes: 2 + workers: 1 + +# Long AgentX prefills can exceed Dynamo's request-plane default while +# the healthy DEP8 workers are still computing their first responses. +override_2p1d: + name: dsv4-gb200-vllm-agentic-mtp-disagg-2p1d-dep8-dep8 + roles: + prefill: + nodes: 4 + workers: 2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml new file mode 100644 index 0000000000..bfe51e70f6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml @@ -0,0 +1,303 @@ +# AgentX dsv4 vllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + dynamo: + install: true + source: + wheel: "1.3.0.dev20260720" + setup_script: vllm-container-deps.sh + slurm: + time_limit: "8:00:00" + health_check: + max_attempts: 2160 + interval_seconds: 10 + resources: + gpu_type: "gb200" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "140GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + engine: + type: vllm + connector: + dp_launch_mode: per_node + roles: + prefill: + gpus: 8 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + workers: 1 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + sbatch_directives: + cpus-per-task: "72" + srun_options: + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +# GB200 AgentX MTP3 topology: one DEP8 prefill worker feeds one +# DEP8 decode worker at concurrency 128. +override_1p1d_c128: + name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-c128-mtp2-agentic" + roles: + prefill: + nodes: 2 + workers: 1 + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c128-{job_id}" + args: + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.85 + decode: + nodes: 2 + gpus: 8 + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c128-{job_id}" + args: + data-parallel-size: 8 + max-num-seqs: 128 + max-num-batched-tokens: 1024 + max-cudagraph-capture-size: 512 + gpu-memory-utilization: 0.92 + +# GB200 AgentX MTP3 topology: one DEP8 prefill worker feeds one +# DEP8 decode worker at concurrency 256. +override_1p1d_c256: + name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-c256-mtp2-agentic" + roles: + prefill: + nodes: 2 + workers: 1 + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c256-{job_id}" + args: + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.85 + decode: + nodes: 2 + gpus: 8 + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c256-{job_id}" + args: + data-parallel-size: 8 + max-num-seqs: 128 + max-num-batched-tokens: 1024 + max-cudagraph-capture-size: 512 + gpu-memory-utilization: 0.92 + +# GB200 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP16 decode worker at concurrency 512. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. +override_2p1d_dep16_c512: + name: "svf-vllm-disagg-gb200-2p1d-dep8-dep16-c512-mtp2-agentic" + roles: + prefill: + nodes: 4 + workers: 2 + env: + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-prefill-{job_id}" + args: + gpu-memory-utilization: 0.90 + decode: + nodes: 4 + gpus: 16 + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-dep16-decode-{job_id}" + args: + data-parallel-size: 16 + max-num-seqs: 64 + max-num-batched-tokens: 256 + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.90 + sbatch_directives: + mem: "0" + +# GB200 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP12 decode worker at concurrency 576. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. +override_2p1d_dep12_c576: + name: "svf-vllm-disagg-gb200-2p1d-dep8-dep12-c576-mtp2-agentic" + roles: + prefill: + nodes: 4 + workers: 2 + env: + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-prefill-{job_id}" + args: + gpu-memory-utilization: 0.90 + decode: + nodes: 3 + gpus: 12 + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-dep12-decode-{job_id}" + args: + data-parallel-size: 12 + max-num-seqs: 64 + max-num-batched-tokens: 256 + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.90 + sbatch_directives: + mem: "0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml deleted file mode 100644 index dda2443295..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml +++ /dev/null @@ -1,159 +0,0 @@ -schema: 2 -name: "svf-vllm-agg-gb300-tp4-mtp-agentic" - -# GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node -# and serves both prefill and decode at concurrency 8. Size max-num-seqs at -# 4x concurrency and expand the MTP CUDA-graph envelope to match. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - install: true - - source: - wheel: "1.2.1" -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -engine: - type: vllm - connector: -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-mtp-{job_id}" - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml deleted file mode 100644 index f17faead2a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml +++ /dev/null @@ -1,160 +0,0 @@ -schema: 2 -name: "svf-vllm-agg-gb300-tp8-mtp-agentic" - -# GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU -# nodes and serves both prefill and decode at concurrency 4. Keep at least -# 16 sequence slots and otherwise size the scheduler at 4x concurrency. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - install: true - - source: - wheel: "1.2.1" -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - # The frontend shares Grace CPU capacity with the long TP8 cold start. - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -engine: - type: vllm - connector: -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-mtp-{job_id}" - args: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 16 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep aggregate workers in the multinode result schema so ingestion uses - # the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..69ec5c40ff --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml @@ -0,0 +1,182 @@ +# AgentX dsv4 vllm gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.2.1" + dynamo: + install: true + source: + wheel: "1.2.1" + environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + # The frontend shares Grace CPU capacity with the long TP8 cold start. + ETCD_LEASE_TTL: "7200" + setup_script: vllm-container-deps.sh + slurm: + time_limit: "8:00:00" + health_check: + max_attempts: 2160 + interval_seconds: 10 + resources: + gpu_type: "gb300" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + engine: + type: vllm + connector: + roles: + agg: + workers: 1 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + gpu-memory-utilization: 0.94 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + sbatch_directives: + mem: "0" + srun_options: + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep aggregate workers in the multinode result schema so ingestion uses + # the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +# GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node +# and serves both prefill and decode at concurrency 8. Size max-num-seqs at +# 4x concurrency and expand the MTP CUDA-graph envelope to match. +override_tp4: + name: "svf-vllm-agg-gb300-tp4-mtp-agentic" + roles: + agg: + nodes: 1 + gpus: 4 + env: + VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-mtp-{job_id}" + args: + tensor-parallel-size: 4 + max-num-seqs: 32 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' + sbatch_directives: + cpus-per-task: "72" + benchmark: + env: + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + +# GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU +# nodes and serves both prefill and decode at concurrency 4. Keep at least +# 16 sequence slots and otherwise size the scheduler at 4x concurrency. +override_tp8: + name: "svf-vllm-agg-gb300-tp8-mtp-agentic" + roles: + agg: + nodes: 2 + gpus: 8 + env: + VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-mtp-{job_id}" + args: + tensor-parallel-size: 8 + max-num-seqs: 16 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' + sbatch_directives: + cpus-per-task: "144" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml deleted file mode 100644 index acdca7c858..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml +++ /dev/null @@ -1,228 +0,0 @@ -# Derived from PR #2302's DEP8/DEP32 c256 recipe. -# Halve the prefill/decode topology while preserving its conservative -# 4/16/16 decode limits and isolated JIT cache paths. -schema: 2 -name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c128-mtp-agentic" - -# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one -# DEP16 decode worker at concurrency 128. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.4.0" - -dynamo: - install: true - - source: - wheel: "1.4.0" -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c128-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c128-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 4 - max-num-batched-tokens: 16 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 16 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml deleted file mode 100644 index bb98740ab7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml +++ /dev/null @@ -1,228 +0,0 @@ -# Derived from PR #2302's DEP8/DEP32 c256 recipe. -# Halve the prefill/decode topology while preserving its conservative -# 8/32/32 decode limits and isolated JIT cache paths. -schema: 2 -name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic" - -# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one -# DEP16 decode worker at concurrency 256. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.4.0" - -dynamo: - install: true - - source: - wheel: "1.4.0" -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 8 - max-num-batched-tokens: 32 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 32 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml deleted file mode 100644 index 62867c6566..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml +++ /dev/null @@ -1,221 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic" - -# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one -# DEP8 decode worker at concurrency 256. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - install: true - - source: - wheel: "1.3.0.dev20260720" -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml deleted file mode 100644 index d9d280a6c9..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml +++ /dev/null @@ -1,228 +0,0 @@ -# Source: https://github.com/SemiAnalysisAI/InferenceX/blob/0c33d4615792705ed12bfc204e3a54cfa436cf02/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml -# Runtime and performance arguments follow the current DEP8/DEP16 GB300 P/D -# recipes; throughput-only synthetic MTP acceptance is injected at launch. -schema: 2 -name: "svf-vllm-disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic" - -# GB300 AgentX MTP3 topology: one DEP8 prefill worker feeds one -# DEP16 decode worker at concurrency 512. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.4.0" - -dynamo: - install: true - - source: - wheel: "1.4.0" -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c512-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 16384 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep16-decode-c512-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml index 922a5343a8..02c44990af 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml @@ -191,7 +191,7 @@ benchmark: # The AgentX client uses localhost:8000, so colocate it with the Dynamo # frontend launched on the first decode node. type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml deleted file mode 100644 index 2670488be1..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb300-1p6d-dep4-tp4-agentic" - -# Agentic-coding variant of dsv4/vllm/gb300-fp4/8k1k/disagg-1p6d-dep4-tp4-stp.yaml. -# Topology is identical (1 prefill DEP=4 + 6 decode TP=4, 28 GPUs across 7 -# GB300 nodes + 1 dedicated NATS/etcd infra node) so we can compare against -# the fixed-seq-len 1p6d baseline at the same concurrency point (192). -# -# Divergence vs the 8k1k sibling: -# - benchmark.type: sa-bench -> custom (hands off to agentic_srt.sh) -# - max-model-len: removed (let vLLM derive from model config; agentic -# trajectories blow past any small explicit cap) -# - no-enable-prefix-caching: dropped (prefix caching MUST be on for -# trajectory reuse — entire point of agentic) -# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser -# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't -# accept them (different arg parser than `vllm serve`). In disagg, chat -# parsing happens at the dynamo frontend, not at the worker. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.21.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - source: - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - attention-config: '{"use_fp4_indexer_cache": true}' - moe-backend: "deep_gemm_mega_moe" - # enforce-eager: true - # max-num-seqs: 256 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.9 - enable-ep-weight-filter: true - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - decode: - nodes: 6 - workers: 6 - gpus: 4 - - env: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - # max-num-seqs: 512 - trust-remote-code: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - enable-ep-weight-filter: true - all2all-backend: "flashinfer_nvlink_one_sided" - no-enable-flashinfer-autotune: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - -# sbatch + srun resource grants for clusters without per-GPU defaults. -# -# mem=0: allocate all available node memory (~868 GB on CW gb300). Without -# this, sbatch only requests ntasks × DefMemPerCPU = 8 × 4 GB = 32 GB for -# the whole job and worker cgroups OOM-kill mid model load (R7-R11 hit -# this; sacct showed AllocTRES mem=4G per step). -# -# cpus-per-task=72: give each task one CW gb300 NUMA socket (144 cores -# split 2 × 72). Critical for the *infra step* (etcd + nats) which -# srtctl spawns without --gres=gpu — on CW that means DefMemPerCPU -# applies and the step gets 1 CPU by default. With 24 dynamo DP ranks -# all hammering etcd for lease keep-alives, single-CPU etcd can't keep -# up and dies (R12 hit this; etcd reported max-cpu-set=1, leases -# deadline-exceeded, infra SIGKILL'd at 16:35:49). 72 CPUs is plenty -# for both etcd + nats AND for vLLM worker auxiliary threads. -# -# nv gb300 doesn't need this because cluster default DefCpuPerGPU=35 -# auto-allocates 4*35=140 CPUs per GPU-bearing task; cw has no per-GPU -# default. Setting it here is safe on both because the value is ≤ node -# CPU count. -# -# srun_options.mem=0 forces each srun step to use the full node memory -# (without it, srun steps default back to cpus_per_task × DefMemPerCPU). -# Docs: docs/config-reference.md#sbatch_directives + #srun_options. -sbatch_directives: - mem: "0" - cpus-per-task: "72" -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user (sa-shared) into the container as - # uid 345200007. dpkg refuses to run without EUID 0 even though - # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the agentic_srt - # apt-get install git step fails. --container-remap-root asks pyxis to - # remap us to uid 0 inside the container. srt-slurm renders empty-string - # values as flag-only srun args (see core/slurm.py:250). - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - # Container-side path of the aiperf mmap dataset cache; the host-side - # mount is wired via launch_gb300-*.sh's srtslurm.yaml default_mounts. - # Without this, aiperf re-tokenizes + re-writes ~65 GB of mmap files - # per dataset on every run. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also wired via default_mounts) so the trace - # dataset isn't re-downloaded on every run. Overrides the workflow-level - # HF_HUB_CACHE=/mnt/hf_hub_cache, which doesn't exist on these nodes. - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml deleted file mode 100644 index 2b234884ff..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml +++ /dev/null @@ -1,223 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic" - -# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one -# DEP12 decode worker at concurrency 1152. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - install: true - - source: - wheel: "1.3.0.dev20260720" -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 4 - workers: 2 - gpus: 8 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 3 - workers: 1 - gpus: 12 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep12-decode-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 12 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml deleted file mode 100644 index 7b8b762e6f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml +++ /dev/null @@ -1,223 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic" - -# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one -# DEP16 decode worker at concurrency 1024. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - install: true - - source: - wheel: "1.3.0.dev20260720" -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 4 - workers: 2 - gpus: 8 - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep16-decode-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - - args: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml deleted file mode 100644 index 20e836b93b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml +++ /dev/null @@ -1,188 +0,0 @@ -schema: 2 -name: "svf-vllm-disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic" - -# Agentic-coding variant of dsv4/vllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep8-24-c4096-stp.yaml. -# Max-throughput shape: 4 prefill (DEP=4 each) + 1 decode (DEP=8). 6 GB300 -# nodes (4P + 2D = 24 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra -# node. Sized for concurrency 4096 with deep_gemm_mega_moe on both workers. -# -# Divergence vs the 8k1k sibling: -# - benchmark.type: sa-bench -> custom (hands off to agentic_srt.sh) -# - max-model-len: removed (let vLLM derive from model config; agentic -# trajectories blow past any small explicit cap) -# - no-enable-prefix-caching: dropped (prefix caching MUST be on for -# trajectory reuse — entire point of agentic) -# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser -# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't -# accept them (different arg parser than `vllm serve`). In disagg, chat -# parsing happens at the dynamo frontend, not at the worker. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.21.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - source: - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 4 - workers: 4 - gpus: 4 - env: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - # enforce-eager: true - # Inherited from fixed-sequence recipes; let vLLM select the scheduler - # sequence limit until this is tuned explicitly for the agentic trace. - # max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-flashinfer-autotune: true - safetensors-load-strategy: "prefetch" - block-size: 256 - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_mega_moe" - - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - # max-num-seqs: 512 - trust-remote-code: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_mega_moe" - -# sbatch + srun resource grants for clusters without per-GPU defaults. -# -# mem=0: allocate all available node memory (~868 GB on CW gb300). Without -# this, sbatch only requests ntasks × DefMemPerCPU = 8 × 4 GB = 32 GB for -# the whole job and worker cgroups OOM-kill mid model load (R7-R11 hit -# this; sacct showed AllocTRES mem=4G per step). -# -# cpus-per-task=72: give each task one CW gb300 NUMA socket (144 cores -# split 2 × 72). Critical for the *infra step* (etcd + nats) which -# srtctl spawns without --gres=gpu — on CW that means DefMemPerCPU -# applies and the step gets 1 CPU by default. With 24 dynamo DP ranks -# all hammering etcd for lease keep-alives, single-CPU etcd can't keep -# up and dies (R12 hit this; etcd reported max-cpu-set=1, leases -# deadline-exceeded, infra SIGKILL'd at 16:35:49). 72 CPUs is plenty -# for both etcd + nats AND for vLLM worker auxiliary threads. -# -# nv gb300 doesn't need this because cluster default DefCpuPerGPU=35 -# auto-allocates 4*35=140 CPUs per GPU-bearing task; cw has no per-GPU -# default. Setting it here is safe on both because the value is ≤ node -# CPU count. -# -# srun_options.mem=0 forces each srun step to use the full node memory -# (without it, srun steps default back to cpus_per_task × DefMemPerCPU). -# Docs: docs/config-reference.md#sbatch_directives + #srun_options. -sbatch_directives: - mem: "0" - cpus-per-task: "72" -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user (sa-shared) into the container as - # uid 345200007. dpkg refuses to run without EUID 0 even though - # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the agentic_srt - # apt-get install git step fails. --container-remap-root asks pyxis to - # remap us to uid 0 inside the container. srt-slurm renders empty-string - # values as flag-only srun args (see core/slurm.py:250). - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - # Container-side path of the aiperf mmap dataset cache; the host-side - # mount is wired via launch_gb300-*.sh's srtslurm.yaml default_mounts. - # Without this, aiperf re-tokenizes + re-writes ~65 GB of mmap files - # per dataset on every run. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also wired via default_mounts) so the trace - # dataset isn't re-downloaded on every run. Overrides the workflow-level - # HF_HUB_CACHE=/mnt/hf_hub_cache, which doesn't exist on these nodes. - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml new file mode 100644 index 0000000000..c91f01d08a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml @@ -0,0 +1,414 @@ +# AgentX dsv4 vllm gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + dynamo: + install: true + environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + setup_script: vllm-container-deps.sh + slurm: + time_limit: "8:00:00" + health_check: + max_attempts: 2160 + interval_seconds: 10 + resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + engine: + type: vllm + connector: + dp_launch_mode: per_node + roles: + prefill: + workers: 1 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + sbatch_directives: + cpus-per-task: "72" + srun_options: + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +# Derived from PR #2302's DEP8/DEP32 c256 recipe. +# Halve the prefill/decode topology while preserving its conservative +# 4/16/16 decode limits and isolated JIT cache paths. +# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one +# DEP16 decode worker at concurrency 128. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. +override_1p1d_dep4_dep16_c128: + name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c128-mtp-agentic" + identity: + frameworks: + dynamo: "1.4.0" + dynamo: + source: + wheel: "1.4.0" + roles: + prefill: + nodes: 1 + gpus: 4 + env: + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c128-{job_id}" + args: + safetensors-load-strategy: "prefetch" + data-parallel-size: 4 + max-num-seqs: 256 + decode: + env: + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c128-{job_id}" + args: + safetensors-load-strategy: "prefetch" + max-num-seqs: 4 + max-num-batched-tokens: 16 + max-cudagraph-capture-size: 16 + sbatch_directives: + mem: "0" + +# Derived from PR #2302's DEP8/DEP32 c256 recipe. +# Halve the prefill/decode topology while preserving its conservative +# 8/32/32 decode limits and isolated JIT cache paths. +# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one +# DEP16 decode worker at concurrency 256. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. +override_1p1d_dep4_dep16_c256: + name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic" + identity: + frameworks: + dynamo: "1.4.0" + dynamo: + source: + wheel: "1.4.0" + roles: + prefill: + nodes: 1 + gpus: 4 + env: + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c256-{job_id}" + args: + safetensors-load-strategy: "prefetch" + data-parallel-size: 4 + max-num-seqs: 256 + decode: + env: + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c256-{job_id}" + args: + safetensors-load-strategy: "prefetch" + max-num-seqs: 8 + max-num-batched-tokens: 32 + max-cudagraph-capture-size: 32 + sbatch_directives: + mem: "0" + +# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one +# DEP8 decode worker at concurrency 256. +override_1p1d_dep4_dep8_c256: + name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic" + identity: + frameworks: + dynamo: "1.3.0.dev20260720" + dynamo: + source: + wheel: "1.3.0.dev20260720" + roles: + prefill: + nodes: 1 + gpus: 4 + env: + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" + args: + data-parallel-size: 4 + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.92 + decode: + nodes: 2 + gpus: 8 + env: + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" + args: + data-parallel-size: 8 + max-num-seqs: 256 + max-num-batched-tokens: 1024 + max-cudagraph-capture-size: 1024 + gpu-memory-utilization: 0.92 + +# Source: https://github.com/SemiAnalysisAI/InferenceX/blob/0c33d4615792705ed12bfc204e3a54cfa436cf02/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml +# Runtime and performance arguments follow the current DEP8/DEP16 GB300 P/D +# recipes; throughput-only synthetic MTP acceptance is injected at launch. +# GB300 AgentX MTP3 topology: one DEP8 prefill worker feeds one +# DEP16 decode worker at concurrency 512. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. +override_1p1d_dep8_dep16_c512: + name: "svf-vllm-disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic" + identity: + frameworks: + dynamo: "1.4.0" + dynamo: + source: + wheel: "1.4.0" + roles: + prefill: + nodes: 2 + gpus: 8 + env: + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c512-{job_id}" + args: + safetensors-load-strategy: "prefetch" + data-parallel-size: 8 + max-num-batched-tokens: 16384 + decode: + env: + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep16-decode-c512-{job_id}" + args: + safetensors-load-strategy: "prefetch" + max-num-seqs: 64 + max-num-batched-tokens: 256 + max-cudagraph-capture-size: 256 + sbatch_directives: + mem: "0" + +# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP16 decode worker at concurrency 1024. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. +override_2p1d_dep8_dep16_c1024: + name: "svf-vllm-disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic" + identity: + frameworks: + dynamo: "1.3.0.dev20260720" + dynamo: + source: + wheel: "1.3.0.dev20260720" + roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + env: + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" + args: + data-parallel-size: 8 + decode: + env: + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep16-decode-{job_id}" + args: + max-num-seqs: 64 + max-num-batched-tokens: 256 + max-cudagraph-capture-size: 256 + sbatch_directives: + mem: "0" + +# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP12 decode worker at concurrency 1152. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. +override_2p1d_dep8_dep12_c1152: + name: "svf-vllm-disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic" + identity: + frameworks: + dynamo: "1.3.0.dev20260720" + dynamo: + source: + wheel: "1.3.0.dev20260720" + roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + env: + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" + args: + data-parallel-size: 8 + decode: + nodes: 3 + gpus: 12 + env: + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep12-decode-{job_id}" + args: + data-parallel-size: 12 + max-num-seqs: 64 + max-num-batched-tokens: 256 + max-cudagraph-capture-size: 256 + sbatch_directives: + mem: "0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..fc0aae1a06 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,222 @@ +# AgentX dsv4 vllm gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0-ubuntu2404" + precision: "fp4" + dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + setup_script: vllm-container-deps.sh + slurm: + time_limit: "8:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: "gb300" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 + frontend: + type: dynamo + enable_multiple_frontends: false + engine: + type: vllm + connector: + roles: + prefill: + gpus: 4 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + moe-backend: "deep_gemm_mega_moe" + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + enable-ep-weight-filter: true + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + decode: + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + pipeline-parallel-size: 1 + # max-num-seqs: 512 + trust-remote-code: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + enable-ep-weight-filter: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + # sbatch + srun resource grants for clusters without per-GPU defaults. + # + # mem=0: allocate all available node memory (~868 GB on CW gb300). Without + # this, sbatch only requests ntasks × DefMemPerCPU = 8 × 4 GB = 32 GB for + # the whole job and worker cgroups OOM-kill mid model load (R7-R11 hit + # this; sacct showed AllocTRES mem=4G per step). + # + # cpus-per-task=72: give each task one CW gb300 NUMA socket (144 cores + # split 2 × 72). Critical for the *infra step* (etcd + nats) which + # srtctl spawns without --gres=gpu — on CW that means DefMemPerCPU + # applies and the step gets 1 CPU by default. With 24 dynamo DP ranks + # all hammering etcd for lease keep-alives, single-CPU etcd can't keep + # up and dies (R12 hit this; etcd reported max-cpu-set=1, leases + # deadline-exceeded, infra SIGKILL'd at 16:35:49). 72 CPUs is plenty + # for both etcd + nats AND for vLLM worker auxiliary threads. + # + # nv gb300 doesn't need this because cluster default DefCpuPerGPU=35 + # auto-allocates 4*35=140 CPUs per GPU-bearing task; cw has no per-GPU + # default. Setting it here is safe on both because the value is ≤ node + # CPU count. + # + # srun_options.mem=0 forces each srun step to use the full node memory + # (without it, srun steps default back to cpus_per_task × DefMemPerCPU). + # Docs: docs/config-reference.md#sbatch_directives + #srun_options. + sbatch_directives: + mem: "0" + cpus-per-task: "72" + srun_options: + mem: "0" + # gb300-nv: pyxis maps the calling user (sa-shared) into the container as + # uid 345200007. dpkg refuses to run without EUID 0 even though + # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the srt_agentic + # apt-get install git step fails. --container-remap-root asks pyxis to + # remap us to uid 0 inside the container. srt-slurm renders empty-string + # values as flag-only srun args (see core/slurm.py:250). + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + # Container-side path of the aiperf mmap dataset cache; the host-side + # mount is wired via launch_gb300-*.sh's srtslurm.yaml default_mounts. + # Without this, aiperf re-tokenizes + re-writes ~65 GB of mmap files + # per dataset on every run. + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + # Persistent HF hub cache (also wired via default_mounts) so the trace + # dataset isn't re-downloaded on every run. Overrides the workflow-level + # HF_HUB_CACHE=/mnt/hf_hub_cache, which doesn't exist on these nodes. + HF_HUB_CACHE: "/hf_hub_cache" + +# Agentic-coding variant of dsv4/vllm/gb300-fp4/8k1k/disagg-1p6d-dep4-tp4-stp.yaml. +# Topology is identical (1 prefill DEP=4 + 6 decode TP=4, 28 GPUs across 7 +# GB300 nodes + 1 dedicated NATS/etcd infra node) so we can compare against +# the fixed-seq-len 1p6d baseline at the same concurrency point (192). +# +# Divergence vs the 8k1k sibling: +# - benchmark.type: sa-bench -> custom (hands off to srt_agentic.sh) +# - max-model-len: removed (let vLLM derive from model config; agentic +# trajectories blow past any small explicit cap) +# - no-enable-prefix-caching: dropped (prefix caching MUST be on for +# trajectory reuse — entire point of agentic) +# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser +# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't +# accept them (different arg parser than `vllm serve`). In disagg, chat +# parsing happens at the dynamo frontend, not at the worker. +override_1p6d_dep4_tp4: + name: "svf-vllm-disagg-gb300-1p6d-dep4-tp4-agentic" + roles: + prefill: + nodes: 1 + workers: 1 + args: + # enforce-eager: true + # max-num-seqs: 256 + attention-config: '{"use_fp4_indexer_cache": true}' + decode: + nodes: 6 + workers: 6 + gpus: 4 + args: + tensor-parallel-size: 4 + all2all-backend: "flashinfer_nvlink_one_sided" + no-enable-flashinfer-autotune: true + +# Agentic-coding variant of dsv4/vllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep8-24-c4096-stp.yaml. +# Max-throughput shape: 4 prefill (DEP=4 each) + 1 decode (DEP=8). 6 GB300 +# nodes (4P + 2D = 24 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra +# node. Sized for concurrency 4096 with deep_gemm_mega_moe on both workers. +# +# Divergence vs the 8k1k sibling: +# - benchmark.type: sa-bench -> custom (hands off to srt_agentic.sh) +# - max-model-len: removed (let vLLM derive from model config; agentic +# trajectories blow past any small explicit cap) +# - no-enable-prefix-caching: dropped (prefix caching MUST be on for +# trajectory reuse — entire point of agentic) +# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser +# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't +# accept them (different arg parser than `vllm serve`). In disagg, chat +# parsing happens at the dynamo frontend, not at the worker. +override_4p1d_dep4_dep8_24_c4096: + name: "svf-vllm-disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic" + roles: + prefill: + nodes: 4 + workers: 4 + env: + VLLM_LOG_STATS_INTERVAL: "1" + args: + # enforce-eager: true + # Inherited from fixed-sequence recipes; let vLLM select the scheduler + # sequence limit until this is tuned explicitly for the agentic trace. + # max-num-seqs: 16 + safetensors-load-strategy: "prefetch" + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + VLLM_LOG_STATS_INTERVAL: "1" + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + moe-backend: "deep_gemm_mega_moe" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/tilert/b200-fp8/agentx/disagg-1p1d-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/tilert/b200-fp8/agentx/disagg-1p1d-tp8-mtp.yaml index 1fff29118f..1d31398dc9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/tilert/b200-fp8/agentx/disagg-1p1d-tp8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/tilert/b200-fp8/agentx/disagg-1p1d-tp8-mtp.yaml @@ -87,7 +87,7 @@ benchmark: mkdir -p /infmax-workspace/LOGS/agentic export RESULT_DIR=/infmax-workspace/LOGS/agentic export AGENTIC_OUTPUT_DIR=/infmax-workspace - exec bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + exec bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml deleted file mode 100644 index 92cd251488..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml +++ /dev/null @@ -1,124 +0,0 @@ -# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 1. -# -# One TP8 worker serves prefill and decode on a single node. The sweep matrix -# supplies the concurrency list; agentic_srt.sh replays every point against -# this one server. Acceptance is pinned to the golden thinking-on AL for three -# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). -schema: 2 -name: agg-b200-tp8-c1-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 - frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 -resources: - gpu_type: b200 - gpus_per_node: 8 -dynamo: - install: false -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 2 - cuda-graph-max-bs-decode: 2 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 110 - hicache-io-backend: direct - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -health_check: - max_attempts: 1440 - interval_seconds: 10 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -frontend: - type: dynamo - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-session-affinity-ttl-secs: 3600 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml deleted file mode 100644 index d28d641d54..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml +++ /dev/null @@ -1,124 +0,0 @@ -# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 4. -# -# One TP8 worker serves prefill and decode on a single node. The sweep matrix -# supplies the concurrency list; agentic_srt.sh replays every point against -# this one server. Acceptance is pinned to the golden thinking-on AL for three -# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). -schema: 2 -name: agg-b200-tp8-c4-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 - frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 -resources: - gpu_type: b200 - gpus_per_node: 8 -dynamo: - install: false -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 8 - cuda-graph-max-bs-decode: 8 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 110 - hicache-io-backend: direct - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -health_check: - max_attempts: 1440 - interval_seconds: 10 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -frontend: - type: dynamo - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-session-affinity-ttl-secs: 3600 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml deleted file mode 100644 index 6b89ba241b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml +++ /dev/null @@ -1,124 +0,0 @@ -# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 8. -# -# One TP8 worker serves prefill and decode on a single node. The sweep matrix -# supplies the concurrency list; agentic_srt.sh replays every point against -# this one server. Acceptance is pinned to the golden thinking-on AL for three -# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). -schema: 2 -name: agg-b200-tp8-c8-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 - frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 -resources: - gpu_type: b200 - gpus_per_node: 8 -dynamo: - install: false -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 110 - hicache-io-backend: direct - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -health_check: - max_attempts: 1440 - interval_seconds: 10 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -frontend: - type: dynamo - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-session-affinity-ttl-secs: 3600 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..f7b88919e4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml @@ -0,0 +1,163 @@ +# AgentX glm5.2 sglang b200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 + identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + frameworks: + dynamo: 1.5.0.dev20260909 + sglang: 0.0.0.dev1+g008403017 + resources: + gpu_type: b200 + gpus_per_node: 8 + dynamo: + install: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 110 + hicache-io-backend: direct + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + health_check: + max_attempts: 1440 + interval_seconds: 10 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 + frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 + sbatch_directives: + mem: '0' + srun_options: + mem: '0' + container-remap-root: '' + +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 1. +# +# One TP8 worker serves prefill and decode on a single node. The sweep matrix +# supplies the concurrency list; srt_agentic.sh replays every point against +# this one server. Acceptance is pinned to the golden thinking-on AL for three +# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). +override_c1: + name: agg-b200-tp8-c1-mtp + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 4. +# +# One TP8 worker serves prefill and decode on a single node. The sweep matrix +# supplies the concurrency list; srt_agentic.sh replays every point against +# this one server. Acceptance is pinned to the golden thinking-on AL for three +# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). +override_c4: + name: agg-b200-tp8-c4-mtp + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 8. +# +# One TP8 worker serves prefill and decode on a single node. The sweep matrix +# supplies the concurrency list; srt_agentic.sh replays every point against +# this one server. Acceptance is pinned to the golden thinking-on AL for three +# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). +override_c8: + name: agg-b200-tp8-c8-mtp + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml deleted file mode 100644 index dd875b3708..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml +++ /dev/null @@ -1,198 +0,0 @@ -schema: 2 -name: disagg-b200-1p1d-dep8-dep8-c64-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 - frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 -resources: - gpu_type: b200 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: false -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 8192 - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 110 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true - decode: - nodes: 1 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 128 - cuda-graph-max-bs-decode: 128 - chunked-prefill-size: 64 - context-length: 1048576 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutedsl - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.85 - disaggregation-decode-extra-slots: 0 - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true - deepep-config: /configs/deepep_config.json - deepep-mode: low_latency - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - speculative-moe-a2a-backend: deepep - speculative-moe-runner-backend: deep_gemm -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml deleted file mode 100644 index ac814c39a1..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml +++ /dev/null @@ -1,190 +0,0 @@ -schema: 2 -name: disagg-b200-1p4d-dep8-tp4-c48-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 - frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 -resources: - gpu_type: b200 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: false -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '3600' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 110 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true - decode: - nodes: 2 - workers: 4 - gpus: 4 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '3600' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 - chunked-prefill-size: 64 - context-length: 1048576 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.9 - disaggregation-decode-extra-slots: 0 - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..02a84e30e1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,238 @@ +# AgentX glm5.2 sglang b200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 + identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + frameworks: + dynamo: 1.5.0.dev20260909 + sglang: 0.0.0.dev1+g008403017 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + dynamo: + install: false + engine: sglang + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 110 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + decode: + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + chunked-prefill-size: 64 + context-length: 1048576 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + disaggregation-decode-extra-slots: 0 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + health_check: + max_attempts: 1440 + interval_seconds: 10 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 + sbatch_directives: + mem: '0' + srun_options: + mem: '0' + container-remap-root: '' + +override_1p4d_tp4_c48: + name: disagg-b200-1p4d-dep8-tp4-c48-mtp + roles: + prefill: + env: + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '3600' + args: + max-prefill-tokens: 16384 + decode: + nodes: 2 + workers: 4 + gpus: 4 + env: + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '3600' + args: + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + moe-runner-backend: flashinfer_trtllm + mem-fraction-static: 0.9 + +override_1p1d_c64: + name: disagg-b200-1p1d-dep8-dep8-c64-mtp + roles: + prefill: + env: + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' + args: + max-prefill-tokens: 8192 + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' + args: + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 128 + cuda-graph-max-bs-decode: 128 + moe-runner-backend: flashinfer_cutedsl + mem-fraction-static: 0.85 + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + speculative-moe-a2a-backend: deepep + speculative-moe-runner-backend: deep_gemm diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml new file mode 100644 index 0000000000..a9c3b602b3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml @@ -0,0 +1,142 @@ +# AgentX glm5.2 sglang gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml:override_. + +schema: 2 + +base: + model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 + identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + frameworks: + dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + sglang: nightly-dev-cu13-20260805-211ee642 + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + dynamo: + install: true + source: + rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + engine: sglang + roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 10 + cuda-graph-max-bs: 10 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 4 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 5 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 135 + hicache-io-backend: direct + enable-metrics: true + health_check: + max_attempts: 1440 + interval_seconds: 10 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + +override_c2: + name: agg-gb200-tp8-c2-mtp + +override_c4: + name: agg-gb200-tp8-c4-mtp + +override_c8: + name: agg-gb200-tp8-c8-mtp diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml deleted file mode 100644 index c56e03e9be..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml +++ /dev/null @@ -1,128 +0,0 @@ -schema: 2 -name: agg-gb200-tp8-c2-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: true - source: - rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd -engine: sglang -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 10 - cuda-graph-max-bs: 10 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - enable-metrics: true -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml deleted file mode 100644 index d845ec11e4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml +++ /dev/null @@ -1,128 +0,0 @@ -schema: 2 -name: agg-gb200-tp8-c4-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: true - source: - rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd -engine: sglang -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 10 - cuda-graph-max-bs: 10 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - enable-metrics: true -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml deleted file mode 100644 index 6b45a9c09e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml +++ /dev/null @@ -1,128 +0,0 @@ -schema: 2 -name: agg-gb200-tp8-c8-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: true - source: - rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd -engine: sglang -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 10 - cuda-graph-max-bs: 10 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - enable-metrics: true -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml index 2be342ac98..3f2650c0f7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml @@ -115,7 +115,7 @@ telemetry: benchmark: type: custom concurrencies: [1] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml deleted file mode 100644 index 27721263a6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml +++ /dev/null @@ -1,199 +0,0 @@ -schema: 2 -name: disagg-gb200-1p4d-dep8-tp4-c48-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: true - source: - rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd -engine: sglang -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - max-running-requests: 16 - cuda-graph-max-bs: 16 - disable-cuda-graph: true - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 2 - enable-metrics: true - enable-cache-report: true - decode: - nodes: 4 - workers: 4 - gpus: 4 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 16 - cuda-graph-max-bs: 16 - chunked-prefill-size: 64 - context-length: 1048576 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.9 - disaggregation-decode-extra-slots: 0 - enable-metrics: true - enable-cache-report: true -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml deleted file mode 100644 index 51f260e38c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml +++ /dev/null @@ -1,199 +0,0 @@ -schema: 2 -name: disagg-gb200-1p6d-dep8-tp4-c45-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: true - source: - rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd -engine: sglang -roles: - prefill: - nodes: 2 - workers: 1 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - max-running-requests: 16 - cuda-graph-max-bs: 16 - disable-cuda-graph: true - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 2 - enable-metrics: true - enable-cache-report: true - decode: - nodes: 6 - workers: 6 - gpus: 4 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 16 - cuda-graph-max-bs: 16 - chunked-prefill-size: 64 - context-length: 1048576 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.9 - disaggregation-decode-extra-slots: 0 - enable-metrics: true - enable-cache-report: true -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml deleted file mode 100644 index dcb66f55d2..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml +++ /dev/null @@ -1,207 +0,0 @@ -schema: 2 -name: disagg-gb200-2p1d-dep8-dep16-c128-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: true - source: - rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd -engine: sglang -roles: - prefill: - nodes: 4 - workers: 2 - gpus: 8 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 8192 - max-running-requests: 16 - cuda-graph-max-bs: 16 - disable-cuda-graph: true - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 100 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 2 - enable-metrics: true - enable-cache-report: true - decode: - nodes: 4 - workers: 1 - gpus: 16 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - args: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 144 - cuda-graph-max-bs: 144 - chunked-prefill-size: 64 - context-length: 1048576 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutedsl - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.85 - disaggregation-decode-extra-slots: 0 - enable-metrics: true - enable-cache-report: true - deepep-config: /configs/deepep_config.json - deepep-mode: low_latency - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - speculative-moe-a2a-backend: deepep - speculative-moe-runner-backend: deep_gemm -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml new file mode 100644 index 0000000000..6411d5880e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml @@ -0,0 +1,247 @@ +# AgentX glm5.2 sglang gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml:override_. + +schema: 2 + +base: + model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 + identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + frameworks: + dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + sglang: nightly-dev-cu13-20260805-211ee642 + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + dynamo: + install: true + source: + rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + engine: sglang + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + max-running-requests: 16 + cuda-graph-max-bs: 16 + disable-cuda-graph: true + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 135 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + enable-metrics: true + enable-cache-report: true + decode: + nodes: 4 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 16 + cuda-graph-max-bs: 16 + chunked-prefill-size: 64 + context-length: 1048576 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.9 + disaggregation-decode-extra-slots: 0 + enable-metrics: true + enable-cache-report: true + health_check: + max_attempts: 1440 + interval_seconds: 10 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + +override_1p6d_tp4_c45: + name: disagg-gb200-1p6d-dep8-tp4-c45-mtp + roles: + decode: + nodes: 6 + workers: 6 + +override_1p4d_tp4_c48: + name: disagg-gb200-1p4d-dep8-tp4-c48-mtp + roles: + decode: + workers: 4 + +override_2p1d_dep16_c128: + name: disagg-gb200-2p1d-dep8-dep16-c128-mtp + roles: + prefill: + nodes: 4 + workers: 2 + args: + max-prefill-tokens: 8192 + hicache-size: 100 + decode: + workers: 1 + gpus: 16 + args: + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 144 + cuda-graph-max-bs: 144 + moe-runner-backend: flashinfer_cutedsl + mem-fraction-static: 0.85 + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + speculative-moe-a2a-backend: deepep + speculative-moe-runner-backend: deep_gemm diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml index 9bf160a4c8..ecf94a8d60 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml @@ -182,7 +182,7 @@ base: interval_seconds: 10 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg.yaml index 6a70bf63e3..72d65baae7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg.yaml @@ -4,7 +4,7 @@ # Flat single-variant schema the agentic CI flow expects: applied via # CONFIG_FILE= with no zip_override selector, so it must NOT be # base:-wrapped. Concurrency is not a recipe field; the GHA matrix fans out -# one job per concurrency from the master-config conc-list into agentic_srt.sh. +# one job per concurrency from the master-config conc-list into srt_agentic.sh. schema: 2 name: gb300-fp4-glm5.2-agentx-agg model: @@ -97,7 +97,7 @@ health_check: interval_seconds: 10 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml index e9d5b35e34..36ef2c32ff 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml @@ -154,7 +154,7 @@ base: interval_seconds: 10 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c12.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c12.yaml index 6595f70ae7..70c86fd033 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c12.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c12.yaml @@ -118,7 +118,7 @@ telemetry: benchmark: type: custom use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c16.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c16.yaml index 682d82606d..c3ee74a2e9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c16.yaml @@ -118,7 +118,7 @@ telemetry: benchmark: type: custom use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c2.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c2.yaml index a96f5d480a..3d2b8a423d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c2.yaml @@ -118,7 +118,7 @@ telemetry: benchmark: type: custom use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c4.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c4.yaml index 22d878c993..60f9736aa1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c4.yaml @@ -118,7 +118,7 @@ telemetry: benchmark: type: custom use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c8.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c8.yaml index 4a9e502c78..a4a603a264 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c8.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c8.yaml @@ -118,7 +118,7 @@ telemetry: benchmark: type: custom use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml index 974b066c92..c8ace70283 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml @@ -152,14 +152,13 @@ telemetry: required: true startup_timeout_seconds: 120 request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 dcgm_exporter: container_image: dcgm-exporter port: 9401 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml index c234617ca4..f50ceb85af 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml @@ -159,7 +159,7 @@ benchmark: type: custom # Replaced with CONC_LIST by the launcher before submission. concurrencies: [1] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml deleted file mode 100644 index 2d0380aec2..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml +++ /dev/null @@ -1,188 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - args: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - tensor_parallel_size: 4 - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: *server_environment - args: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1, 2, 4, 5] - enable_padding: true - enable_attention_dp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - tokens_per_block: 64 - max_batch_size: 5 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - stream_interval: 20 - tensor_parallel_size: 8 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: head -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - placement: - node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml deleted file mode 100644 index dcf151af95..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml +++ /dev/null @@ -1,185 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - args: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - disable_overlap_scheduler: true - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: EQUAL_PROGRESS - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - tensor_parallel_size: 4 - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: *server_environment - args: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1] - enable_padding: true - enable_attention_dp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - tokens_per_block: 64 - max_batch_size: 1 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 1 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - stream_interval: 20 - tensor_parallel_size: 8 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: head -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - placement: - node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml deleted file mode 100644 index 9408847d11..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml +++ /dev/null @@ -1,188 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - args: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - tensor_parallel_size: 4 - decode: - nodes: 4 - workers: 4 - gpus: 4 - - env: *server_environment - args: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1, 2] - enable_padding: true - enable_attention_dp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - tokens_per_block: 64 - max_batch_size: 2 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - stream_interval: 20 - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: head -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - placement: - node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml deleted file mode 100644 index 507a1c08eb..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml +++ /dev/null @@ -1,188 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false - -roles: - prefill: - nodes: 3 - workers: 3 - gpus: 4 - env: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - args: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - tensor_parallel_size: 4 - decode: - nodes: 4 - workers: 4 - gpus: 4 - - env: *server_environment - args: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1, 2, 4, 5] - enable_padding: true - enable_attention_dp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - tokens_per_block: 64 - max_batch_size: 5 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - stream_interval: 20 - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: head -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - placement: - node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml deleted file mode 100644 index bb733a601e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml +++ /dev/null @@ -1,189 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-4p1d-dep8-c227-b16-mtp3 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false - -roles: - prefill: - nodes: 4 - workers: 4 - gpus: 4 - env: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - args: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 4 - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: *server_environment - args: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1, 2, 4, 8, 16] - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.9 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 16 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 8 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: head -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - placement: - node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml deleted file mode 100644 index 7a9647373a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml +++ /dev/null @@ -1,189 +0,0 @@ -schema: 2 -name: dynamo-disagg-gb300-5p1d-dep16-c260-b16-mtp3 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false - -roles: - prefill: - nodes: 5 - workers: 5 - gpus: 4 - env: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - args: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 4 - decode: - nodes: 4 - workers: 1 - gpus: 16 - - env: *server_environment - args: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1, 2, 4, 8, 16] - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.9 - tokens_per_block: 64 - max_batch_size: 16 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 16 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - placement: - node: head -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - placement: - node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..c8e5cfd844 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,401 @@ +# AgentX glm5.2 trtllm gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + precision: fp4 + dynamo: + install: true + request_plane: tcp + health_check: + max_attempts: 270 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + engine: + type: trtllm + publish_events_and_metrics: false + roles: + prefill: + gpus: 4 + env: + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_FUSED_DSA_METADATA: "1" + TRTLLM_DSA_INDEXER_BF16: "1" + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + args: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + tensor_parallel_size: 4 + decode: + workers: 1 + env: + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_FUSED_DSA_METADATA: "1" + TRTLLM_DSA_INDEXER_BF16: "1" + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + args: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: head + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + placement: + node: dedicated + +override_1p1d_tp8_c1_b1_mtp5: + name: dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5 + dynamo: + source: + wheel: 1.4.0.dev20260807 + roles: + prefill: + nodes: 1 + workers: 1 + args: + disable_overlap_scheduler: true + scheduler_config: + context_chunking_policy: EQUAL_PROGRESS + decode: + nodes: 2 + gpus: 8 + args: + cuda_graph_config: + batch_sizes: + - 1 + max_batch_size: 1 + moe_expert_parallel_size: 1 + sparse_attention_config: + use_cute_dsl_topk: true + tensor_parallel_size: 8 + +override_1p1d_tep8_c20_b5_mtp5: + name: dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5 + dynamo: + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + roles: + prefill: + nodes: 1 + workers: 1 + args: + enable_iter_perf_stats: true + decode: + nodes: 2 + gpus: 8 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 5 + max_batch_size: 5 + moe_expert_parallel_size: 8 + enable_iter_perf_stats: true + sparse_attention_config: + use_cute_dsl_topk: true + tensor_parallel_size: 8 + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + +override_1p4d_tep4_c30_b2_mtp5: + name: dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5 + dynamo: + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + roles: + prefill: + nodes: 1 + workers: 1 + args: + enable_iter_perf_stats: true + decode: + nodes: 4 + workers: 4 + gpus: 4 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + max_batch_size: 2 + moe_expert_parallel_size: 4 + enable_iter_perf_stats: true + sparse_attention_config: + use_cute_dsl_topk: true + tensor_parallel_size: 4 + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + +override_3p4d_tep4_c60_b5_mtp5: + name: dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5 + dynamo: + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + roles: + prefill: + nodes: 3 + workers: 3 + args: + enable_iter_perf_stats: true + decode: + nodes: 4 + workers: 4 + gpus: 4 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 5 + max_batch_size: 5 + moe_expert_parallel_size: 4 + enable_iter_perf_stats: true + sparse_attention_config: + use_cute_dsl_topk: true + tensor_parallel_size: 4 + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + +override_4p1d_dep8_c227_b16_mtp3: + name: dynamo-disagg-gb300-4p1d-dep8-c227-b16-mtp3 + dynamo: + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + roles: + prefill: + nodes: 4 + workers: 4 + args: + enable_iter_perf_stats: true + speculative_config: + max_draft_len: 3 + decode: + nodes: 2 + gpus: 8 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + free_gpu_memory_fraction: 0.9 + host_cache_size: 137438953472 + max_batch_size: 16 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 8 + enable_iter_perf_stats: true + speculative_config: + max_draft_len: 3 + tensor_parallel_size: 8 + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + +override_5p1d_dep16_c260_b16_mtp3: + name: dynamo-disagg-gb300-5p1d-dep16-c260-b16-mtp3 + dynamo: + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + roles: + prefill: + nodes: 5 + workers: 5 + args: + enable_iter_perf_stats: true + speculative_config: + max_draft_len: 3 + decode: + nodes: 4 + gpus: 16 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + free_gpu_memory_fraction: 0.9 + max_batch_size: 16 + moe_expert_parallel_size: 16 + enable_iter_perf_stats: true + sparse_attention_config: + use_cute_dsl_topk: true + speculative_config: + max_draft_len: 3 + tensor_parallel_size: 16 + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml index 532de487ee..ad4390d7e0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml @@ -127,7 +127,7 @@ telemetry: benchmark: type: custom concurrencies: [1] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml index 590cfc7f7e..fed72d219b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml @@ -127,7 +127,7 @@ telemetry: benchmark: type: custom concurrencies: [14] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml index cdcb3fb8d0..ccd6894f55 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml @@ -127,7 +127,7 @@ telemetry: benchmark: type: custom concurrencies: [24] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml index a45f184bec..5dd98d53a6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml @@ -127,7 +127,7 @@ telemetry: benchmark: type: custom concurrencies: [4] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml index a8d44175ac..250c4a5b35 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml @@ -127,7 +127,7 @@ telemetry: benchmark: type: custom concurrencies: [48] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml index 2bff214dd2..c2cffc57b4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml @@ -127,7 +127,7 @@ telemetry: benchmark: type: custom concurrencies: [8] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml index 28018a5910..b9d2212953 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml @@ -126,7 +126,7 @@ telemetry: benchmark: type: custom concurrencies: [96] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml index ceb0efb874..fa1ef92c37 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml @@ -159,7 +159,7 @@ benchmark: type: custom client_placement: head concurrencies: [1, 2, 4, 8, 16] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml index 81af8a3005..ed5eef9320 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml @@ -155,7 +155,7 @@ benchmark: type: custom client_placement: head concurrencies: [8, 40, 48] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml deleted file mode 100644 index ff19c7103b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml +++ /dev/null @@ -1,163 +0,0 @@ -schema: 2 -name: "kimi-k3-vllm-agg-gb200-dep16-throughput-vllm-simple-offload-agentic" - -# High-concurrency host-DRAM KV-offload variant of the official throughput- -# oriented multi_node_dep profile. TP4 x DP4 gives EP16 across four four-GPU -# GB200 nodes. Each TP rank receives a 128 GiB CPU KV pool (512 GiB per node). -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_dep - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - install: true - - source: - rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "12:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - args: - # Use vLLM's Kimi K3 parser so OpenAI responses expose structured - # tool_calls instead of raw XTML in message.content. - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 1 - kv-cache-block-size: 64 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - NVIDIA_GDRCOPY: "1" - PYTHONHASHSEED: "42" - PYTORCH_ALLOC_CONF: "expandable_segments:True" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-dep16-offload-{job_id}" - args: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - # FlashInfer's larger TP4 MoE representation leaves too little transient - # HBM for fastsafetensors' GPU-staging path when DSpark is loaded. - load-format: "safetensors" - safetensors-load-strategy: "lazy" - kv-cache-dtype: "fp8" - attention-backend: "FLASHINFER_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - # DeepGEMM mega-MoE grid barriers time out under DSpark MRV2 with the - # one-sided all-to-all worker; use the supported Kimi K3 FlashInfer path. - moe-backend: "flashinfer_trtllm" - kda-prefill-backend: "flashkda" - kernel-config: '{"enable_cutedsl_warmup":true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.94 - # The largest offload point is c384 / DP4 = 96 sequences per engine. - # Capture even sequence counts: all configured DP4 steady-state batch - # sizes are exact hits, while odd loads pad by at most one sequence. - max-num-seqs: 96 - max-num-batched-tokens: 16384 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96,102,108,114,120,126,132,138,144,150,156,162,168,174,180,186,192,198,204,210,216,222,228,234,240,246,252,258,264,270,276,282,288]}' - block-size: 64 - language-model-only: true - disable-custom-all-reduce: true - enable-prefix-caching: true - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' - scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" - dyn-tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - dyn-reasoning-parser: "kimi_k3" - no-enable-flashinfer-autotune: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AGENTIC_WARMUP_GRACE_PERIOD: "3600" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml deleted file mode 100644 index 4edaabdfc5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml +++ /dev/null @@ -1,160 +0,0 @@ -schema: 2 -name: "kimi-k3-vllm-agg-gb200-dep16-throughput-agentic" - -# Day-0 GB200 translation of the official throughput-oriented multi_node_dep -# profile. TP4 x DP4 gives EP16 across four four-GPU GB200 nodes. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_dep - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - install: true - - source: - rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "12:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - args: - # Use vLLM's Kimi K3 parser so OpenAI responses expose structured - # tool_calls instead of raw XTML in message.content. - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 1 - kv-cache-block-size: 64 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - NVIDIA_GDRCOPY: "1" - PYTORCH_ALLOC_CONF: "expandable_segments:True" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-dep16-{job_id}" - args: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - # FlashInfer's larger TP4 MoE representation leaves too little transient - # HBM for fastsafetensors' GPU-staging path when DSpark is loaded. - load-format: "safetensors" - safetensors-load-strategy: "lazy" - kv-cache-dtype: "fp8" - attention-backend: "FLASHINFER_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - # DeepGEMM mega-MoE grid barriers time out under DSpark MRV2 with the - # one-sided all-to-all worker; use the supported Kimi K3 FlashInfer path. - moe-backend: "flashinfer_trtllm" - kda-prefill-backend: "flashkda" - kernel-config: '{"enable_cutedsl_warmup":true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.94 - # The largest regular DEP point is c256 / DP4 = 64 sequences per engine. - # Capturing 128 sequence slots consumes 10.9 GiB and leaves too little - # runtime workspace for FlashInfer's MXFP4 MoE kernel. - max-num-seqs: 64 - max-num-batched-tokens: 16384 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96,99,102,105,108,111,114,117,120,123,126,129,132,135,138,141,144,147,150,153,156,159,162,165,168,171,174,177,180,183,186,189,192]}' - block-size: 64 - language-model-only: true - disable-custom-all-reduce: true - enable-prefix-caching: true - scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" - dyn-tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - dyn-reasoning-parser: "kimi_k3" - no-enable-flashinfer-autotune: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AGENTIC_WARMUP_GRACE_PERIOD: "3600" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml deleted file mode 100644 index aace6df184..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml +++ /dev/null @@ -1,151 +0,0 @@ -schema: 2 -name: "kimi-k3-vllm-agg-gb200-tep16-balanced-agentic" - -# Day-0 GB200 translation of the official balanced multi_node_tep profile. -# Dense layers and MoE experts are sharded across 16 GPUs on four GB200 nodes -# with the official FP8 KV cache. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tep - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - install: true - - source: - rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - args: - # Use vLLM's Kimi K3 parser so OpenAI responses expose structured - # tool_calls instead of raw XTML in message.content. - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 1 - kv-cache-block-size: 64 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - NVIDIA_GDRCOPY: "1" - PYTORCH_ALLOC_CONF: "expandable_segments:True" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-tep16-{job_id}" - args: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 - pipeline-parallel-size: 1 - enable-expert-parallel: true - trust-remote-code: true - load-format: "fastsafetensors" - safetensors-load-strategy: "lazy" - kv-cache-dtype: "fp8" - attention-backend: "FLASHINFER_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - moe-backend: "flashinfer_trtllm" - kda-prefill-backend: "flashkda" - kernel-config: '{"enable_cutedsl_warmup":true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.92 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96],"pass_config":{"fuse_allreduce_rms":false}}' - block-size: 64 - language-model-only: true - disable-custom-all-reduce: true - enable-prefix-caching: true - scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" - dyn-tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - dyn-reasoning-parser: "kimi_k3" - no-enable-flashinfer-autotune: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml deleted file mode 100644 index a838dc8f73..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml +++ /dev/null @@ -1,149 +0,0 @@ -schema: 2 -name: "kimi-k3-vllm-agg-gb200-tp16-latency-agentic" - -# Day-0 GB200 translation of the official latency-oriented multi_node_tp -# profile. TP16 spans four GB200 nodes and uses the official FP8 KV cache. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tp - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - install: true - - source: - rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 -frontend: - type: dynamo - enable_multiple_frontends: false - args: - # Use vLLM's Kimi K3 parser so OpenAI responses expose structured - # tool_calls instead of raw XTML in message.content. - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 1 - kv-cache-block-size: 64 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 4 - workers: 1 - gpus: 16 - - env: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - NVIDIA_GDRCOPY: "1" - PYTORCH_ALLOC_CONF: "expandable_segments:True" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-tp16-{job_id}" - args: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 - pipeline-parallel-size: 1 - trust-remote-code: true - load-format: "fastsafetensors" - safetensors-load-strategy: "lazy" - kv-cache-dtype: "fp8" - attention-backend: "FLASHINFER_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - moe-backend: "flashinfer_trtllm" - kda-prefill-backend: "flashkda" - kernel-config: '{"enable_cutedsl_warmup":true}' - gpu-memory-utilization: 0.92 - max-num-seqs: 8 - max-num-batched-tokens: 8192 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24],"pass_config":{"fuse_allreduce_rms":false}}' - block-size: 64 - language-model-only: true - disable-custom-all-reduce: true - enable-prefix-caching: true - scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" - dyn-tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - dyn-reasoning-parser: "kimi_k3" - no-enable-flashinfer-autotune: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml index 33f3115102..8346ab2963 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml @@ -128,7 +128,7 @@ benchmark: type: custom client_placement: head concurrencies: [16] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml index cf98995eec..a6cfad55f0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml @@ -128,7 +128,7 @@ benchmark: type: custom client_placement: head concurrencies: [32] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml index 57498c8969..f86e4d3119 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml @@ -127,7 +127,7 @@ benchmark: type: custom client_placement: head concurrencies: [48] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml index c7e1da7937..7ae1438a5d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml @@ -127,7 +127,7 @@ benchmark: type: custom client_placement: head concurrencies: [72] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml index 2bb53eaee4..930d9425d0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml @@ -127,7 +127,7 @@ benchmark: type: custom client_placement: head concurrencies: [96] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..4afb5e8a62 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml @@ -0,0 +1,238 @@ +# AgentX kimik3 vllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "kimi-k3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0" + dynamo: + install: true + source: + rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + setup_script: kimik3-dspark-config-compat.sh + environment: + ETCD_LEASE_TTL: "7200" + health_check: + max_attempts: 2160 + interval_seconds: 10 + resources: + gpu_type: "gb200" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + enable_multiple_frontends: false + args: + # Use vLLM's Kimi K3 parser so OpenAI responses expose structured + # tool_calls instead of raw XTML in message.content. + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + enable-auto-tool-choice: true + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-min-initial-workers: 1 + kv-cache-block-size: 64 + engine: + type: vllm + connector: + roles: + agg: + nodes: 4 + workers: 1 + gpus: 16 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + NVIDIA_GDRCOPY: "1" + PYTORCH_ALLOC_CONF: "expandable_segments:True" + args: + served-model-name: "moonshotai/Kimi-K3" + pipeline-parallel-size: 1 + trust-remote-code: true + safetensors-load-strategy: "lazy" + kv-cache-dtype: "fp8" + attention-backend: "FLASHINFER_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # DeepGEMM mega-MoE grid barriers time out under DSpark MRV2 with the + # one-sided all-to-all worker; use the supported Kimi K3 FlashInfer path. + moe-backend: "flashinfer_trtllm" + kda-prefill-backend: "flashkda" + kernel-config: '{"enable_cutedsl_warmup":true}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + block-size: 64 + language-model-only: true + disable-custom-all-reduce: true + enable-prefix-caching: true + scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" + dyn-tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + dyn-reasoning-parser: "kimi_k3" + no-enable-flashinfer-autotune: true + sbatch_directives: + cpus-per-task: "144" + mem: "0" + srun_options: + container-remap-root: "" + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +# Day-0 GB200 translation of the official latency-oriented multi_node_tp +# profile. TP16 spans four GB200 nodes and uses the official FP8 KV cache. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tp +override_tp16_latency: + name: "kimi-k3-vllm-agg-gb200-tp16-latency-agentic" + slurm: + time_limit: "8:00:00" + roles: + agg: + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-tp16-{job_id}" + args: + tensor-parallel-size: 16 + load-format: "fastsafetensors" + gpu-memory-utilization: 0.92 + max-num-seqs: 8 + max-num-batched-tokens: 8192 + compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24],"pass_config":{"fuse_allreduce_rms":false}}' + benchmark: + env: + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + +# Day-0 GB200 translation of the official balanced multi_node_tep profile. +# Dense layers and MoE experts are sharded across 16 GPUs on four GB200 nodes +# with the official FP8 KV cache. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tep +override_tep16_balanced: + name: "kimi-k3-vllm-agg-gb200-tep16-balanced-agentic" + slurm: + time_limit: "8:00:00" + roles: + agg: + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-tep16-{job_id}" + args: + tensor-parallel-size: 16 + enable-expert-parallel: true + load-format: "fastsafetensors" + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.92 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96],"pass_config":{"fuse_allreduce_rms":false}}' + +# Day-0 GB200 translation of the official throughput-oriented multi_node_dep +# profile. TP4 x DP4 gives EP16 across four four-GPU GB200 nodes. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_dep +override_dep16: + name: "kimi-k3-vllm-agg-gb200-dep16-throughput-agentic" + slurm: + time_limit: "12:00:00" + roles: + agg: + env: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-dep16-{job_id}" + args: + tensor-parallel-size: 4 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + # FlashInfer's larger TP4 MoE representation leaves too little transient + # HBM for fastsafetensors' GPU-staging path when DSpark is loaded. + load-format: "safetensors" + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.94 + # The largest regular DEP point is c256 / DP4 = 64 sequences per engine. + # Capturing 128 sequence slots consumes 10.9 GiB and leaves too little + # runtime workspace for FlashInfer's MXFP4 MoE kernel. + max-num-seqs: 64 + max-num-batched-tokens: 16384 + compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96,99,102,105,108,111,114,117,120,123,126,129,132,135,138,141,144,147,150,153,156,159,162,165,168,171,174,177,180,183,186,189,192]}' + benchmark: + env: + AGENTIC_WARMUP_GRACE_PERIOD: "3600" + +# High-concurrency host-DRAM KV-offload variant of the official throughput- +# oriented multi_node_dep profile. TP4 x DP4 gives EP16 across four four-GPU +# GB200 nodes. Each TP rank receives a 128 GiB CPU KV pool (512 GiB per node). +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_dep +override_dep16_vllm_simple_offload: + name: "kimi-k3-vllm-agg-gb200-dep16-throughput-vllm-simple-offload-agentic" + slurm: + time_limit: "12:00:00" + roles: + agg: + env: + PYTHONHASHSEED: "42" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-dep16-offload-{job_id}" + args: + tensor-parallel-size: 4 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + # FlashInfer's larger TP4 MoE representation leaves too little transient + # HBM for fastsafetensors' GPU-staging path when DSpark is loaded. + load-format: "safetensors" + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.94 + # The largest offload point is c384 / DP4 = 96 sequences per engine. + # Capture even sequence counts: all configured DP4 steady-state batch + # sizes are exact hits, while odd loads pad by at most one sequence. + max-num-seqs: 96 + max-num-batched-tokens: 16384 + compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96,102,108,114,120,126,132,138,144,150,156,162,168,174,180,186,192,198,204,210,216,222,228,234,240,246,252,258,264,270,276,282,288]}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' + benchmark: + env: + AGENTIC_WARMUP_GRACE_PERIOD: "3600" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml index bac87abe62..a15277f843 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml @@ -145,7 +145,7 @@ telemetry: benchmark: type: custom concurrencies: [48, 52, 56] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml index 113a947f37..a921e677a0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml @@ -143,7 +143,7 @@ telemetry: benchmark: type: custom concurrencies: [1, 4] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml index dd9a30f069..0c77c8cb70 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml @@ -174,8 +174,7 @@ srun_options: telemetry: enabled: true - provider: dcgm-power - default_frequency: 1.0 + collect_interval_ms: 1000 storage_subdir: power required: true startup_timeout_seconds: 120 @@ -189,7 +188,7 @@ benchmark: type: custom client_placement: head concurrencies: [48, 52, 56] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml index 6ea49208c6..d640ad21d2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml @@ -174,8 +174,7 @@ srun_options: telemetry: enabled: true - provider: dcgm-power - default_frequency: 1.0 + collect_interval_ms: 1000 storage_subdir: power required: true startup_timeout_seconds: 120 @@ -189,7 +188,7 @@ benchmark: type: custom client_placement: head concurrencies: [32, 48, 64] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml index 4bf4ecc656..9565c64c8e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml @@ -174,8 +174,7 @@ srun_options: telemetry: enabled: true - provider: dcgm-power - default_frequency: 1.0 + collect_interval_ms: 1000 storage_subdir: power required: true startup_timeout_seconds: 120 @@ -189,7 +188,7 @@ benchmark: type: custom client_placement: head concurrencies: [32, 48] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml index 0cd2a16883..92dbd01512 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml @@ -174,8 +174,7 @@ srun_options: telemetry: enabled: true - provider: dcgm-power - default_frequency: 1.0 + collect_interval_ms: 1000 storage_subdir: power required: true startup_timeout_seconds: 120 @@ -189,7 +188,7 @@ benchmark: type: custom client_placement: head concurrencies: [1] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml index 0ae9ca0fe6..d5b84a386d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml @@ -97,7 +97,7 @@ telemetry: benchmark: type: custom concurrencies: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml index 6c241ec561..ed0ce03355 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml @@ -85,8 +85,7 @@ srun_options: telemetry: enabled: true - provider: dcgm-power - default_frequency: 1.0 + collect_interval_ms: 1000 storage_subdir: power required: true startup_timeout_seconds: 120 @@ -99,7 +98,7 @@ telemetry: benchmark: type: custom concurrencies: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml index edfb3d9963..249925f675 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml @@ -86,8 +86,7 @@ srun_options: telemetry: enabled: true - provider: dcgm-power - default_frequency: 1.0 + collect_interval_ms: 1000 storage_subdir: power required: true startup_timeout_seconds: 120 @@ -100,7 +99,7 @@ telemetry: benchmark: type: custom concurrencies: [8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml deleted file mode 100644 index c9574016ba..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml +++ /dev/null @@ -1,152 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=10 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# AgentX acceptance is selected from the committed golden curve at submission. -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -schema: 2 -name: dynamo-agg-gb200-tp4-c10-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 -engine: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - publish_events_and_metrics: false -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - args: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 10 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '10' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml deleted file mode 100644 index cef285921e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml +++ /dev/null @@ -1,157 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=15 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# AgentX acceptance is selected from the committed golden curve at submission. -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -schema: 2 -name: dynamo-agg-gb200-tp4-c15-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 -engine: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - publish_events_and_metrics: false -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - args: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 15 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 13 - - 14 - - 15 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '15' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml deleted file mode 100644 index 55c9e38641..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml +++ /dev/null @@ -1,162 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=20 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# AgentX acceptance is selected from the committed golden curve at submission. -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -schema: 2 -name: dynamo-agg-gb200-tp4-c20-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 -engine: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - publish_events_and_metrics: false -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - args: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 20 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 13 - - 14 - - 15 - - 16 - - 17 - - 18 - - 19 - - 20 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '20' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml deleted file mode 100644 index 25e2a5df3e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml +++ /dev/null @@ -1,162 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=25 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# AgentX acceptance is selected from the committed golden curve at submission. -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -schema: 2 -name: dynamo-agg-gb200-tp4-c25-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 -engine: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - publish_events_and_metrics: false -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - args: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 25 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 13 - - 14 - - 15 - - 17 - - 19 - - 21 - - 23 - - 25 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '25' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml deleted file mode 100644 index b6315eaa71..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml +++ /dev/null @@ -1,162 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=30 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# AgentX acceptance is selected from the committed golden curve at submission. -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -schema: 2 -name: dynamo-agg-gb200-tp4-c30-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 -engine: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - publish_events_and_metrics: false -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - args: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 30 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 14 - - 16 - - 18 - - 20 - - 22 - - 24 - - 27 - - 30 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '30' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml deleted file mode 100644 index 1d29eebbc1..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml +++ /dev/null @@ -1,166 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=40 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# AgentX acceptance is selected from the committed golden curve at submission. -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -schema: 2 -name: dynamo-agg-gb200-tp4-c40-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 -engine: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - publish_events_and_metrics: false -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - args: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 40 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 14 - - 16 - - 18 - - 20 - - 22 - - 24 - - 26 - - 28 - - 30 - - 32 - - 34 - - 36 - - 38 - - 40 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '40' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml deleted file mode 100644 index 9279351de5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml +++ /dev/null @@ -1,147 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=5 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# AgentX acceptance is selected from the committed golden curve at submission. -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -schema: 2 -name: dynamo-agg-gb200-tp4-c5-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - source: - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 -engine: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - publish_events_and_metrics: false -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - args: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 5 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '5' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..6ac78e12b7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml @@ -0,0 +1,372 @@ +# AgentX minimaxm3 trtllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: minimax-m3-nvfp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 + precision: fp4 + dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + request_plane: tcp + health_check: + max_attempts: 270 + interval_seconds: 10 + resources: + gpu_type: gb200 + gpus_per_node: 4 + engine: + type: trtllm + numa_memory_bind: true + served_model_name: nvidia/MiniMax-M3-NVFP4 + publish_events_and_metrics: false + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + HF_HUB_CACHE: /hf_hub_cache + TQDM_DISABLE: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + PYTHONNOUSERSITE: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + args: + max_seq_len: 1048576 + max_num_tokens: 16384 + cuda_graph_config: + enable_padding: true + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: + - 1 + - 512 + - 1024 + - 2048 + enable_userbuffers: true + max_num_streams: 3 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + free_gpu_memory_fraction: 0.94 + enable_block_reuse: true + block_reuse_policy: per_conversation + tokens_per_block: 128 + use_kv_cache_manager_v2: true + dtype: fp8 + event_buffer_max_size: 0 + host_cache_size: 137438953472 + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + enable_chunked_prefill: true + enable_autotuner: true + trust_remote_code: true + stream_interval: 20 + print_iter_log: true + num_postprocess_workers: 8 + enable_attention_dp: false + tensor_parallel_size: 4 + frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + KV_OFFLOAD_BACKEND: native + KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' + TOTAL_CPU_DRAM_GB: '550' + MODEL: nvidia/MiniMax-M3-NVFP4 + MODEL_PREFIX: minimaxm3 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + DURATION: '3600' + +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=5 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# AgentX acceptance is selected from the committed golden curve at submission. +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +override_c5_b5: + name: dynamo-agg-gb200-tp4-c5-b1-eagle3 + roles: + agg: + args: + max_batch_size: 5 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + benchmark: + env: + CONC: '5' + +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=10 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# AgentX acceptance is selected from the committed golden curve at submission. +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +override_c10_b10: + name: dynamo-agg-gb200-tp4-c10-b1-eagle3 + roles: + agg: + args: + max_batch_size: 10 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + benchmark: + env: + CONC: '10' + +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=15 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# AgentX acceptance is selected from the committed golden curve at submission. +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +override_c15_b15: + name: dynamo-agg-gb200-tp4-c15-b1-eagle3 + roles: + agg: + args: + max_batch_size: 15 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + benchmark: + env: + CONC: '15' + +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=20 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# AgentX acceptance is selected from the committed golden curve at submission. +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +override_c20_b20: + name: dynamo-agg-gb200-tp4-c20-b1-eagle3 + roles: + agg: + args: + max_batch_size: 20 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + benchmark: + env: + CONC: '20' + +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=25 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# AgentX acceptance is selected from the committed golden curve at submission. +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +override_c25_b25: + name: dynamo-agg-gb200-tp4-c25-b1-eagle3 + roles: + agg: + args: + max_batch_size: 25 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 17 + - 19 + - 21 + - 23 + - 25 + benchmark: + env: + CONC: '25' + +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=30 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# AgentX acceptance is selected from the committed golden curve at submission. +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +override_c30_b30: + name: dynamo-agg-gb200-tp4-c30-b1-eagle3 + roles: + agg: + args: + max_batch_size: 30 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 27 + - 30 + benchmark: + env: + CONC: '30' + +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=40 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# AgentX acceptance is selected from the committed golden curve at submission. +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +override_c40_b40: + name: dynamo-agg-gb200-tp4-c40-b1-eagle3 + roles: + agg: + args: + max_batch_size: 40 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 26 + - 28 + - 30 + - 32 + - 34 + - 36 + - 38 + - 40 + benchmark: + env: + CONC: '40' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4-vllm-simple.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4-vllm-simple.yaml index eb661f2c60..1ec16d4f66 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4-vllm-simple.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4-vllm-simple.yaml @@ -92,7 +92,7 @@ sbatch_directives: {cpus-per-task: "144", mem: "0"} srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4.yaml index 33324c4699..8bb79b49b4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4.yaml @@ -90,7 +90,7 @@ sbatch_directives: {cpus-per-task: "144", mem: "0"} srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep8.yaml index 020dd3d4d0..49fda02e0c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep8.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep8.yaml @@ -90,7 +90,7 @@ sbatch_directives: {cpus-per-task: "144", mem: "0"} srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml new file mode 100644 index 0000000000..bd8c7bc9b6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml @@ -0,0 +1,129 @@ +# AgentX minimaxm3 vllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "minimax-m3-nvfp4" + container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" + precision: "fp4" + identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7"} + frameworks: {dynamo: "1.5.0.dev20260908"} + dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} + environment: {ETCD_LEASE_TTL: "7200"} + slurm: {time_limit: "12:00:00"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: "gb200" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + enable_multiple_frontends: false + env: {DYN_TCP_REQUEST_TIMEOUT: "60"} + args: + trust-remote-code: true + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 + engine: + type: vllm + connector: + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + kv_events: true + sbatch_directives: {cpus-per-task: "144", mem: "0"} + srun_options: {container-remap-root: ""} + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +override_tp4: + name: "minimax-m3-vllm-agg-gb200-tp4-agentic-nightly-native" + +override_tp4_vllm_simple: + name: "minimax-m3-vllm-agg-gb200-tp4-vllm-simple-agentic-nightly-native" + roles: + agg: + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + args: + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":777389080576,"cpu_bytes_to_use_per_rank":194347270144,"lazy_offload":true}}' + +override_tp8: + name: "minimax-m3-vllm-agg-gb200-tp8-agentic-nightly-native" + roles: + agg: + nodes: 2 + gpus: 8 + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + args: + tensor-parallel-size: 8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml deleted file mode 100644 index 5b4a692ed4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml +++ /dev/null @@ -1,111 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-agg-gb200-tp4-agentic-nightly-native" - -model: - path: "minimax-m3-nvfp4" - container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" - precision: "fp4" - -identity: - model: {repo: "nvidia/MiniMax-M3-NVFP4"} - container: {image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7"} - frameworks: {dynamo: "1.5.0.dev20260908"} - -dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} -environment: {ETCD_LEASE_TTL: "7200"} - -slurm: {time_limit: "12:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: {DYN_TCP_REQUEST_TIMEOUT: "60"} - args: - trust-remote-code: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - - env: - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - - kv_events: true -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml deleted file mode 100644 index cd56742486..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml +++ /dev/null @@ -1,108 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-agg-gb200-tp4-vllm-simple-agentic-nightly-native" - -model: - path: "minimax-m3-nvfp4" - container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" - precision: "fp4" - -identity: - model: {repo: "nvidia/MiniMax-M3-NVFP4"} - container: {image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7"} - frameworks: {dynamo: "1.5.0.dev20260908"} - -dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} -environment: {ETCD_LEASE_TTL: "7200"} -slurm: {time_limit: "12:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} -resources: {gpu_type: "gb200", gpus_per_node: 4} -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: {DYN_TCP_REQUEST_TIMEOUT: "60"} - args: - trust-remote-code: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - env: - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_SIMPLE_KV_OFFLOAD: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":777389080576,"cpu_bytes_to_use_per_rank":194347270144,"lazy_offload":true}}' - stream-interval: 20 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - - kv_events: true -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple.yaml index f88160a9cf..54390af54e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple.yaml @@ -97,7 +97,7 @@ srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4.yaml index f8fdb3c152..e956a39828 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4.yaml @@ -100,7 +100,7 @@ srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml deleted file mode 100644 index 590eac5eaf..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml +++ /dev/null @@ -1,111 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-agg-gb200-tp8-agentic-nightly-native" - -model: - path: "minimax-m3-nvfp4" - container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" - precision: "fp4" - -identity: - model: {repo: "nvidia/MiniMax-M3-NVFP4"} - container: {image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7"} - frameworks: {dynamo: "1.5.0.dev20260908"} - -dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} -environment: {ETCD_LEASE_TTL: "7200"} - -slurm: {time_limit: "12:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: {DYN_TCP_REQUEST_TIMEOUT: "60"} - args: - trust-remote-code: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 - -engine: - type: vllm - connector: -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - - env: - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' - stream-interval: 20 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - - kv_events: true -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep4.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep4.yaml index d04d2c6299..d7d6023eaf 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep4.yaml @@ -126,7 +126,7 @@ sbatch_directives: {cpus-per-task: "144", mem: "0"} srun_options: {container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml deleted file mode 100644 index ea049b7320..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml +++ /dev/null @@ -1,173 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb200-1p1d-tp4-tp4-c24-agentic" - -model: - path: "minimax-m3-nvfp4" - container: &container "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6" - precision: "fp4" - -identity: - model: {repo: "nvidia/MiniMax-M3-NVFP4"} - container: {image: *container} - frameworks: {dynamo: "1.5.0.dev20260819"} - -dynamo: {install: true, source: {wheel: "1.5.0.dev20260819"}} -environment: {PYTHONHASHSEED: "0"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 3600 - env: - DYN_LOG: "info" - DYN_TCP_CONNECT_TIMEOUT: "120" - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE: "1" - -engine: - type: vllm - connector: - dp_launch_mode: per_gpu -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &worker_environment - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MEMTYPE_CACHE: "n" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - UCX_TLS: "tcp,cuda_ipc,cuda_copy" - WITH_NVIDIA_PEERMEM: "0" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "1" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: &numa_nodes [0, 0, 1, 1] - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - nodes: 1 - workers: 1 - gpus: 4 - - env: *worker_environment - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: *numa_nodes - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AGENTIC_WARMUP_GRACE_PERIOD: "1800" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "3600" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml deleted file mode 100644 index 2a0121782d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml +++ /dev/null @@ -1,180 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb200-1p1d-tp4-tp8-c1-agentic" - -model: - path: "minimax-m3-nvfp4" - container: &container "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6" - precision: "fp4" - -identity: - model: {repo: "nvidia/MiniMax-M3-NVFP4"} - container: {image: *container} - frameworks: {dynamo: "1.5.0.dev20260819"} - -dynamo: {install: true, source: {wheel: "1.5.0.dev20260819"}} -environment: {PYTHONHASHSEED: "0"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 3600 - env: - DYN_LOG: "info" - DYN_TCP_CONNECT_TIMEOUT: "120" - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE: "1" - -engine: - type: vllm - connector: - dp_launch_mode: per_gpu -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &worker_environment - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MEMTYPE_CACHE: "n" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - UCX_TLS: "tcp,cuda_ipc,cuda_copy" - WITH_NVIDIA_PEERMEM: "0" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "1" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: &numa_nodes [0, 0, 1, 1] - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "1" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 8 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-num-seqs: 1 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: *numa_nodes - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AGENTIC_WARMUP_GRACE_PERIOD: "1800" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "3600" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml deleted file mode 100644 index 513b7a7954..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml +++ /dev/null @@ -1,173 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb200-1p2d-tp4-tp4-c8-c16-agentic" - -model: - path: "minimax-m3-nvfp4" - container: &container "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6" - precision: "fp4" - -identity: - model: {repo: "nvidia/MiniMax-M3-NVFP4"} - container: {image: *container} - frameworks: {dynamo: "1.5.0.dev20260819"} - -dynamo: {install: true, source: {wheel: "1.5.0.dev20260819"}} -environment: {PYTHONHASHSEED: "0"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 3600 - env: - DYN_LOG: "info" - DYN_TCP_CONNECT_TIMEOUT: "120" - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE: "1" - -engine: - type: vllm - connector: - dp_launch_mode: per_gpu -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &worker_environment - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MEMTYPE_CACHE: "n" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - UCX_TLS: "tcp,cuda_ipc,cuda_copy" - WITH_NVIDIA_PEERMEM: "0" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "1" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: &numa_nodes [0, 0, 1, 1] - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - nodes: 2 - workers: 2 - gpus: 4 - - env: *worker_environment - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: *numa_nodes - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - -sbatch_directives: {cpus-per-task: "144", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AGENTIC_WARMUP_GRACE_PERIOD: "1800" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "3600" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..8899d709fc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,218 @@ +# AgentX minimaxm3 vllm gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "minimax-m3-nvfp4" + container: "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6" + precision: "fp4" + identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6"} + frameworks: {dynamo: "1.5.0.dev20260819"} + dynamo: {install: true, source: {wheel: "1.5.0.dev20260819"}} + environment: {PYTHONHASHSEED: "0"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: "gb200" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "minimax_m3" + reasoning-parser: "minimax_m3" + enable-auto-tool-choice: true + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 3600 + env: + DYN_LOG: "info" + DYN_TCP_CONNECT_TIMEOUT: "120" + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE: "1" + engine: + type: vllm + connector: + dp_launch_mode: per_gpu + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + UCX_TLS: "tcp,cuda_ipc,cuda_copy" + WITH_NVIDIA_PEERMEM: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "1" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 128 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + decode: + nodes: 2 + workers: 1 + gpus: 4 + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "1" + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + UCX_TLS: "tcp,cuda_ipc,cuda_copy" + WITH_NVIDIA_PEERMEM: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 128 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + sbatch_directives: {cpus-per-task: "144", mem: "0"} + srun_options: {container-remap-root: ""} + benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "3600" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +override_1p1d_tp8_c1: + name: "minimax-m3-vllm-disagg-gb200-1p1d-tp4-tp8-c1-agentic" + roles: + decode: + gpus: 8 + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + args: + tensor-parallel-size: 8 + max-num-seqs: 1 + +override_1p2d_c8_c16: + name: "minimax-m3-vllm-disagg-gb200-1p2d-tp4-tp4-c8-c16-agentic" + roles: + decode: + workers: 2 + +override_1p1d_c24: + name: "minimax-m3-vllm-disagg-gb200-1p1d-tp4-tp4-c24-agentic" + roles: + decode: + nodes: 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml deleted file mode 100644 index 44e9395766..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-eval-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 1 - workers: 1 - gpus: 4 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml deleted file mode 100644 index d121a66c8b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 1 - workers: 1 - gpus: 4 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml deleted file mode 100644 index 88bc24e782..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-eval-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 1 - workers: 1 - gpus: 4 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml deleted file mode 100644 index 3677f72f71..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 1 - workers: 1 - gpus: 4 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml deleted file mode 100644 index 1e6fadc381..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml +++ /dev/null @@ -1,182 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-eval-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 3 - workers: 3 - gpus: 4 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml deleted file mode 100644 index 7efc02a2ab..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml +++ /dev/null @@ -1,182 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 3 - workers: 3 - gpus: 4 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml deleted file mode 100644 index d0e7aa0acf..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-eval-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: true -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 3 - workers: 3 - gpus: 2 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml deleted file mode 100644 index 142bad0347..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: true -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 3 - workers: 3 - gpus: 2 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml deleted file mode 100644 index 2ca720c02e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-eval-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: true -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 2 - workers: 2 - gpus: 2 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 5 - workers: 5 - gpus: 2 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml deleted file mode 100644 index f980e5f3fa..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml +++ /dev/null @@ -1,179 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - install: true - - source: - pypi: "1.4.0.dev20260730" -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: true -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -engine: - type: vllm - connector: - dp_launch_mode: per_node -roles: - prefill: - nodes: 2 - workers: 2 - gpus: 2 - env: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - nodes: 5 - workers: 5 - gpus: 2 - - env: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..eefda7020c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,364 @@ +# AgentX minimaxm3 vllm gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + dynamo: + install: true + source: + pypi: "1.4.0.dev20260730" + health_check: + max_attempts: 2160 + interval_seconds: 10 + resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + environment: + PYTHONHASHSEED: "0" + frontend: + type: dynamo + enable_multiple_frontends: true + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + engine: + type: vllm + connector: + dp_launch_mode: per_node + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + gpus: 4 + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true + sbatch_directives: + cpus-per-task: "72" + mem: "0" + srun_options: + container-remap-root: "" + benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + +override_1p1d_tep4_tp4_c1: + name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-agentic" + frontend: + enable_multiple_frontends: false + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + decode: + nodes: 1 + workers: 1 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + +override_1p1d_tep4_tp4_c1_eval: + name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-eval-agentic" + frontend: + enable_multiple_frontends: false + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + decode: + nodes: 1 + workers: 1 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + +override_1p1d_tp2_tp4_c20_c24: + name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-agentic" + frontend: + num_additional_frontends: 1 + roles: + prefill: + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + decode: + nodes: 1 + workers: 1 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + +override_1p1d_tp2_tp4_c20_c24_eval: + name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-eval-agentic" + frontend: + num_additional_frontends: 1 + roles: + prefill: + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + decode: + nodes: 1 + workers: 1 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + +override_1p3d_dep4_tp4_c24: + name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-agentic" + frontend: + num_additional_frontends: 1 + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + decode: + nodes: 3 + workers: 3 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + +override_1p3d_dep4_tp4_c24_eval: + name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-eval-agentic" + frontend: + num_additional_frontends: 1 + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + decode: + nodes: 3 + workers: 3 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + +override_1p3d_tp2_tp2_c48: + name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-agentic" + resources: + spread_workers: true + frontend: + num_additional_frontends: 4 + roles: + prefill: + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + decode: + nodes: 3 + workers: 3 + gpus: 2 + args: + tensor-parallel-size: 2 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + +override_1p3d_tp2_tp2_c48_eval: + name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-eval-agentic" + resources: + spread_workers: true + frontend: + num_additional_frontends: 4 + roles: + prefill: + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + decode: + nodes: 3 + workers: 3 + gpus: 2 + args: + tensor-parallel-size: 2 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + +override_2p5d_tp2_tp2_c120: + name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-agentic" + resources: + spread_workers: true + frontend: + num_additional_frontends: 4 + roles: + prefill: + nodes: 2 + workers: 2 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + decode: + nodes: 5 + workers: 5 + gpus: 2 + args: + tensor-parallel-size: 2 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + +override_2p5d_tp2_tp2_c120_eval: + name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-eval-agentic" + resources: + spread_workers: true + frontend: + num_additional_frontends: 4 + roles: + prefill: + nodes: 2 + workers: 2 + args: + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + decode: + nodes: 5 + workers: 5 + gpus: 2 + args: + tensor-parallel-size: 2 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml index a3f8bb681c..61baf8076f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml @@ -154,7 +154,7 @@ telemetry: port: 9401 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh concurrencies: - 16 env: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml index a9af6f1251..0744261c8d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml @@ -154,7 +154,7 @@ telemetry: port: 9401 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh concurrencies: - 24 env: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml index 5dab4917a0..3fcbfe1ba6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml @@ -154,7 +154,7 @@ telemetry: port: 9401 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh concurrencies: - 32 env: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml index 5197d3be5a..96ce0b05f2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml @@ -155,7 +155,7 @@ telemetry: port: 9401 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh concurrencies: - 48 env: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml index ac6a00a1d8..816a5bdf42 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml @@ -155,7 +155,7 @@ telemetry: port: 9401 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh concurrencies: - 64 env: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml index 2492edcae4..11f0f770cb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml @@ -154,7 +154,7 @@ telemetry: port: 9401 benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh concurrencies: - 8 env: diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml deleted file mode 100644 index c73b3f0f1f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated TP2+EP2 prefill and decode with write-through HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - enable-hierarchical-cache: true - hicache-size: 298 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through - decode: - nodes: colocate - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.88 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - cuda-graph-max-bs-decode: 32 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 32 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml deleted file mode 100644 index 0150c174ba..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated TP2+EP2 prefill and decode with write-through HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - enable-hierarchical-cache: true - hicache-size: 298 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through - decode: - nodes: colocate - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.88 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - cuda-graph-max-bs-decode: 48 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 40 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml deleted file mode 100644 index cecb894de7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated TP2+EP2 prefill and decode with write-through HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - enable-hierarchical-cache: true - hicache-size: 298 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through - decode: - nodes: colocate - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.88 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - cuda-graph-max-bs-decode: 48 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 44 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml deleted file mode 100644 index d5b99fbbcd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated TP2+EP2 prefill and decode with write-through HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - enable-hierarchical-cache: true - hicache-size: 298 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through - decode: - nodes: colocate - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.92 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - cuda-graph-max-bs-decode: 48 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 48 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache.yaml deleted file mode 100644 index f007aad45b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache.yaml +++ /dev/null @@ -1,180 +0,0 @@ -# Colocated TP2+EP2 prefill and decode with write-through HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - enable-hierarchical-cache: true - hicache-size: 298 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through - decode: - nodes: colocate - workers: 1 - gpus: 2 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - expert-parallel-size: 2 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.92 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - enable-linear-replayssm-spec: true - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 64 - cuda-graph-max-bs-decode: 48 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 56 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml deleted file mode 100644 index a8d1f5bc62..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated prefill and decode with prefill HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - enable-hierarchical-cache: true - hicache-size: 72 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through_selective - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - cuda-graph-max-bs-decode: 16 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 12 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml deleted file mode 100644 index 1b1fe769ad..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated TP4 prefill and decode with prefill HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - enable-hierarchical-cache: true - hicache-size: 72 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through_selective - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - cuda-graph-max-bs-decode: 32 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 24 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml deleted file mode 100644 index d88c06b458..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated prefill and decode with prefill HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - enable-hierarchical-cache: true - hicache-size: 72 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through_selective - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - cuda-graph-max-bs-decode: 16 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 4 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml deleted file mode 100644 index ce87f0369b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# Colocated prefill and decode with prefill HiCache. -schema: 2 -name: qwen35-b300-disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 -identity: - model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 - revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 -dynamo: - install: true - source: - rev: 805a77f053d807b0d8def5d27f674a6df0ed839e -slurm: - time_limit: '4:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: b300 - gpus_per_node: 8 -frontend: - type: dynamo - nginx_session_affinity: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 4 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: prefill - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - disable-cuda-graph: true - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 2048 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - enable-hierarchical-cache: true - hicache-size: 72 - hicache-io-backend: kernel - hicache-mem-layout: page_first - hicache-write-policy: write_through_selective - decode: - nodes: colocate - workers: 1 - gpus: 4 - env: - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com - NCCL_NVLS_ENABLE: '1' - SGLANG_TIMEOUT_KEEP_ALIVE: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - MC_INTRANODE_NVLINK: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - args: - served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 - model-path: /model/ - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - mamba-ssm-dtype: bfloat16 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - enable-symm-mem: true - enable-cache-report: true - enable-metrics: true - context-length: 262144 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - page-size: 64 - reasoning-parser: qwen3 - tool-call-parser: qwen3_coder - disaggregation-mode: decode - disaggregation-transfer-backend: mooncake - watchdog-timeout: 3600 - mamba-radix-cache-strategy: no_buffer - disable-radix-cache: true - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - scheduler-recv-interval: 10 - stream-interval: 50 - max-running-requests: 32 - cuda-graph-max-bs-decode: 32 - disaggregation-decode-extra-slots: 0 - disaggregation-decode-retraction-backup: cpu_tensor -sbatch_directives: - mem: '0' - cpus-per-task: '144' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - concurrencies: - - 32 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..71394d6ae5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml @@ -0,0 +1,336 @@ +# AgentX qwen3.5 sglang b300-fp8 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: qwen3.5-fp8 + container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + precision: fp8 + identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + revision: ea5b4f81096f3901c91dea97f81324302495781d + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + dynamo: + install: true + source: + rev: 805a77f053d807b0d8def5d27f674a6df0ed839e + slurm: + time_limit: '4:00:00' + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: dynamo + nginx_session_affinity: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID + engine: sglang + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + disable-cuda-graph: true + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 2048 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + enable-hierarchical-cache: true + hicache-size: 298 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through + decode: + nodes: colocate + workers: 1 + gpus: 2 + env: + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PIP_EXTRA_INDEX_URL: https://pypi.nvidia.com + NCCL_NVLS_ENABLE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_INTRANODE_NVLINK: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: INTRA_NODE_NVLINK + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + model-path: /model/ + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + expert-parallel-size: 2 + data-parallel-size: 1 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + enable-symm-mem: true + enable-cache-report: true + enable-metrics: true + context-length: 262144 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + page-size: 64 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + watchdog-timeout: 3600 + mamba-radix-cache-strategy: no_buffer + disable-radix-cache: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + scheduler-recv-interval: 10 + stream-interval: 50 + max-running-requests: 64 + disaggregation-decode-extra-slots: 0 + disaggregation-decode-retraction-backup: cpu_tensor + sbatch_directives: + mem: '0' + cpus-per-task: '144' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + ENABLE_AGENTX_POWER: '0' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# Colocated prefill and decode with prefill HiCache. +override_tp4_tp4_c4: + name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + max-running-requests: 32 + hicache-size: 72 + hicache-write-policy: write_through_selective + decode: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + max-running-requests: 32 + cuda-graph-max-bs-decode: 16 + benchmark: + concurrencies: + - 4 + +# Colocated prefill and decode with prefill HiCache. +override_tp4_tp4_c12: + name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + max-running-requests: 32 + hicache-size: 72 + hicache-write-policy: write_through_selective + decode: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + max-running-requests: 32 + cuda-graph-max-bs-decode: 16 + benchmark: + concurrencies: + - 12 + +# Colocated TP4 prefill and decode with prefill HiCache. +override_tp4_tp4_c24: + name: qwen35-b300-disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + max-running-requests: 32 + hicache-size: 72 + hicache-write-policy: write_through_selective + decode: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + benchmark: + concurrencies: + - 24 + +# Colocated prefill and decode with prefill HiCache. +override_tp4ep4_tp4_c32: + name: qwen35-b300-disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache + roles: + prefill: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + max-running-requests: 32 + hicache-size: 72 + hicache-write-policy: write_through_selective + decode: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + benchmark: + concurrencies: + - 32 + +# Colocated TP2+EP2 prefill and decode with write-through HiCache. +override_tp2ep2_tp2ep2_c32: + name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache + roles: + decode: + args: + mem-fraction-static: 0.88 + cuda-graph-max-bs-decode: 32 + benchmark: + concurrencies: + - 32 + +# Colocated TP2+EP2 prefill and decode with write-through HiCache. +override_tp2ep2_tp2ep2_c40: + name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache + roles: + decode: + args: + mem-fraction-static: 0.88 + cuda-graph-max-bs-decode: 48 + benchmark: + concurrencies: + - 40 + +# Colocated TP2+EP2 prefill and decode with write-through HiCache. +override_tp2ep2_tp2ep2_c44: + name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache + roles: + decode: + args: + mem-fraction-static: 0.88 + cuda-graph-max-bs-decode: 48 + benchmark: + concurrencies: + - 44 + +# Colocated TP2+EP2 prefill and decode with write-through HiCache. +override_tp2ep2_tp2ep2_c48: + name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache + roles: + decode: + args: + mem-fraction-static: 0.92 + cuda-graph-max-bs-decode: 48 + benchmark: + concurrencies: + - 48 + +# Colocated TP2+EP2 prefill and decode with write-through HiCache. +override_tp2ep2_tp2ep2_c56_replayssm: + name: qwen35-b300-disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache + roles: + decode: + args: + mem-fraction-static: 0.92 + enable-linear-replayssm-spec: true + cuda-graph-max-bs-decode: 48 + benchmark: + concurrencies: + - 56 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml deleted file mode 100644 index 6912536759..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml +++ /dev/null @@ -1,115 +0,0 @@ -schema: 2 -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-cap48 - -model: {path: qwen3.5-fp4, container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b, precision: fp4} -identity: - model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} - container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} -resources: {gpu_type: gb200, gpus_per_node: 4} -services: - - name: nats - type: nats - options: - max_payload_mb: 8 -dynamo: - install: true - source: - wheel: 1.4.2 - request_plane: tcp -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-radix-cache-strategy: extra_buffer_lazy - mamba-track-interval: 1048576 - attention-backend: trtllm_mha - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 8 - cuda-graph-max-bs: 48 - max-running-requests: 48 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.88 - max-mamba-cache-size: 192 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 2.5 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - hicache-write-policy: write_through_selective - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - watchdog-timeout: 1000000 - enable-metrics: true - enable-cache-report: true - -sbatch_directives: {mem: "0", cpus-per-task: "144"} -srun_options: {mem: "0", container-remap-root: ""} -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "2" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml deleted file mode 100644 index 624e4e7f68..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml +++ /dev/null @@ -1,116 +0,0 @@ -schema: 2 -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-k3-baseline - -model: {path: qwen3.5-fp4, container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b, precision: fp4} -identity: - model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} - container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} -resources: {gpu_type: gb200, gpus_per_node: 4} -services: - - name: nats - type: nats - options: - max_payload_mb: 8 -dynamo: - install: true - source: - wheel: 1.4.2 - request_plane: tcp -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-radix-cache-strategy: extra_buffer_lazy - mamba-track-interval: 1048576 - attention-backend: trtllm_mha - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 8 - cuda-graph-max-bs: 64 - max-running-requests: 80 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.88 - max-mamba-cache-size: 320 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 1 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 2.5 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - hicache-write-policy: write_through_selective - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - watchdog-timeout: 1000000 - enable-metrics: true - enable-cache-report: true - -sbatch_directives: {mem: "0", cpus-per-task: "144"} -srun_options: {mem: "0", container-remap-root: ""} -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "2" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml deleted file mode 100644 index 35991d1a61..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml +++ /dev/null @@ -1,117 +0,0 @@ -schema: 2 -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-k5 - -model: {path: qwen3.5-fp4, container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b, precision: fp4} -identity: - model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} - container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} -resources: {gpu_type: gb200, gpus_per_node: 4} -services: - - name: nats - type: nats - options: - max_payload_mb: 8 -dynamo: - install: true - source: - wheel: 1.4.2 - request_plane: tcp -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-radix-cache-strategy: extra_buffer_lazy - mamba-track-interval: 1048576 - attention-backend: trtllm_mha - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 8 - cuda-graph-max-bs: 64 - max-running-requests: 80 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.88 - # extra_buffer_lazy uses four physical Mamba state slots per running request. - # This layout is the measured K5 middle frontier through C28. - max-mamba-cache-size: 320 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 2.5 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - hicache-write-policy: write_through_selective - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - watchdog-timeout: 1000000 - enable-metrics: true - enable-cache-report: true - -sbatch_directives: {mem: "0", cpus-per-task: "144"} -srun_options: {mem: "0", container-remap-root: ""} -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "2" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-hicache.yaml index eab7f2107d..c0077e00fb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-hicache.yaml @@ -73,7 +73,7 @@ sbatch_directives: {mem: "0", cpus-per-task: "144"} srun_options: {mem: "0", container-remap-root: ""} benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml deleted file mode 100644 index db3e9116f2..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml +++ /dev/null @@ -1,111 +0,0 @@ -schema: 2 -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5-no-symm - -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b - precision: fp4 - -identity: - model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} - container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} - -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: gb200 - gpus_per_node: 4 -services: - - name: nats - type: nats - options: - max_payload_mb: 8 -dynamo: - install: true - source: - wheel: 1.4.2 - request_plane: tcp -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - - env: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: false - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - cuda-graph-max-bs: 64 - max-running-requests: 80 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.80 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - enable-metrics: true - enable-cache-report: true - -sbatch_directives: {mem: "0", cpus-per-task: "144"} -srun_options: {mem: "0", container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "4" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml deleted file mode 100644 index 4697c70678..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml +++ /dev/null @@ -1,114 +0,0 @@ -schema: 2 -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5-parity - -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b - precision: fp4 - -identity: - model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} - container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} - -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: gb200 - gpus_per_node: 4 -services: - - name: nats - type: nats - options: - max_payload_mb: 8 -dynamo: - install: true - source: - wheel: 1.4.2 - request_plane: tcp -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - - env: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer_lazy - mamba-track-interval: 1048576 - attention-backend: trtllm_mha - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 8 - cuda-graph-max-bs: 64 - max-running-requests: 80 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.80 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - enable-metrics: true - enable-cache-report: true - -sbatch_directives: {mem: "0", cpus-per-task: "144"} -srun_options: {mem: "0", container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "4" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml deleted file mode 100644 index ea6ea024e3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml +++ /dev/null @@ -1,111 +0,0 @@ -schema: 2 -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5 - -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b - precision: fp4 - -identity: - model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} - container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} - -slurm: {time_limit: "8:00:00"} -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: gb200 - gpus_per_node: 4 -services: - - name: nats - type: nats - options: - max_payload_mb: 8 -dynamo: - install: true - source: - wheel: 1.4.2 - request_plane: tcp -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 4 - - env: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - cuda-graph-max-bs: 64 - max-running-requests: 80 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.80 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - enable-metrics: true - enable-cache-report: true - -sbatch_directives: {mem: "0", cpus-per-task: "144"} -srun_options: {mem: "0", container-remap-root: ""} - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "4" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..c2e9769d2b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml @@ -0,0 +1,249 @@ +# AgentX qwen3.5 sglang gb200-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b + precision: fp4 + identity: + model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} + container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} + slurm: {time_limit: "8:00:00"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: gb200 + gpus_per_node: 4 + services: + - name: nats + type: nats + options: + max_payload_mb: 8 + dynamo: + install: true + source: + wheel: 1.4.2 + request_plane: tcp + environment: + PIP_BREAK_SYSTEM_PACKAGES: "1" + frontend: + type: dynamo + enable_multiple_frontends: false + env: {} + args: + router-mode: round-robin + router-session-affinity-ttl-secs: 3600 + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + env: + PYTHONNOUSERSITE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_ENABLE_SPEC_V2: "1" + SGL_ENABLE_JIT_DEEPGEMM: "false" + TORCH_CUDA_ARCH_LIST: "10.0" + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model/ + trust-remote-code: true + data-parallel-size: 1 + enable-symm-mem: false + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-track-interval: 1048576 + attention-backend: trtllm_mha + linear-attn-decode-backend: flashinfer + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + cuda-graph-max-bs: 64 + max-running-requests: 80 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + mamba-max-states-per-path: 1 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + weight-loader-drop-cache-after-load: false + model-loader-extra-config: '{"enable_multithread_load":true}' + enable-metrics: true + enable-cache-report: true + sbatch_directives: {mem: "0", cpus-per-task: "144"} + srun_options: {mem: "0", container-remap-root: ""} + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' + +override_tp4: + name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5 + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + enable-symm-mem: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + mem-fraction-static: 0.80 + max-mamba-cache-size: 360 + benchmark: + env: + TP: "4" + +override_tp4_parity: + name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5-parity + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + enable-symm-mem: true + mamba-scheduler-strategy: extra_buffer_lazy + linear-attn-prefill-backend: flashinfer + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 8 + mem-fraction-static: 0.80 + max-mamba-cache-size: 360 + benchmark: + env: + TP: "4" + +override_tp4_no_symm: + name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5-no-symm + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + mem-fraction-static: 0.80 + max-mamba-cache-size: 360 + benchmark: + env: + TP: "4" + +override_tp2ep2_hicache: + name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-k5 + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + expert-parallel-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-radix-cache-strategy: extra_buffer_lazy + linear-attn-prefill-backend: flashinfer + speculative-moe-runner-backend: flashinfer_trtllm + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 8 + mem-fraction-static: 0.88 + # extra_buffer_lazy uses four physical Mamba state slots per running request. + # This layout is the measured K5 middle frontier through C28. + max-mamba-cache-size: 320 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 2.5 + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-write-policy: write_through_selective + watchdog-timeout: 1000000 + benchmark: + env: + TP: "2" + +override_tp2ep2_hicache_cap48: + name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-cap48 + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + expert-parallel-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-radix-cache-strategy: extra_buffer_lazy + linear-attn-prefill-backend: flashinfer + speculative-moe-runner-backend: flashinfer_trtllm + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 8 + cuda-graph-max-bs: 48 + max-running-requests: 48 + mem-fraction-static: 0.88 + max-mamba-cache-size: 192 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 2.5 + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-write-policy: write_through_selective + watchdog-timeout: 1000000 + benchmark: + env: + TP: "2" + +override_tp2ep2_hicache_k3_baseline: + name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-k3-baseline + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + expert-parallel-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-radix-cache-strategy: extra_buffer_lazy + linear-attn-prefill-backend: flashinfer + speculative-moe-runner-backend: flashinfer_trtllm + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 8 + mem-fraction-static: 0.88 + max-mamba-cache-size: 320 + tokenizer-worker-num: 1 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 2.5 + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-write-policy: write_through_selective + watchdog-timeout: 1000000 + benchmark: + env: + TP: "2" diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml new file mode 100644 index 0000000000..c65a482875 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml @@ -0,0 +1,186 @@ +# AgentX qwen3.5 sglang gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml:override_. + +schema: 2 + +base: + model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + precision: fp4 + slurm: + time_limit: '8:00:00' + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 2 + frontend: + type: sglang-router + enable_multiple_frontends: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-dense-tp-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + moe-a2a-backend: none + load-balance-method: round_robin + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 1536 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 6 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + speculative-moe-runner-backend: flashinfer_trtllm + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + mem-fraction-static: 0.85 + max-running-requests: 1 + pp-max-micro-batch-size: 1 + prefill-max-requests: 1 + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-size: 32 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true + cuda-graph-max-bs-decode: 1 + cuda-graph-bs-decode: + - 1 + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +override_tp2_c44_b1: + name: agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831 + roles: + agg: + env: + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/flashinfer-cache + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/sglang-cache + args: + prefill-decode-interval: 0 + +override_tp2_c44_b2: + name: agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831 + roles: + agg: + env: + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/flashinfer-cache + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/sglang-cache + args: + prefill-decode-interval: 0 + max-running-requests: 2 + pp-max-micro-batch-size: 2 + prefill-max-requests: 2 + cuda-graph-max-bs-decode: 2 + cuda-graph-bs-decode: + - 1 + - 2 + hicache-size: 128 + +override_tp8_c7_b1: + name: agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831 + resources: + gpus_per_node: 4 + roles: + agg: + nodes: 2 + gpus: 8 + env: + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/c7-mrr1/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/c7-mrr1/flashinfer-cache + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/c7-mrr1/sglang-cache + args: + tensor-parallel-size: 8 + moe-dense-tp-size: 8 + moe-runner-backend: flashinfer_cutedsl + speculative-moe-runner-backend: flashinfer_cutedsl + disable-prefill-cuda-graph: true + benchmark: + env: + IS_MULTINODE: 'true' + TP: '8' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml deleted file mode 100644 index 1e6f6493fe..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c1-mtp-hicache-jid2530006 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml deleted file mode 100644 index b6579ff4ac..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c24-mtp-hicache-jid2530012 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml deleted file mode 100644 index 0f5a27a447..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c32-mtp-hicache-jid2530013 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml deleted file mode 100644 index 9b3d398808..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c40-mtp-hicache-jid2530015 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml deleted file mode 100644 index 7a5a8417a6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml +++ /dev/null @@ -1,132 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831 -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router - enable_multiple_frontends: false -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/flashinfer-cache - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/sglang-cache - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 1 - moe-dense-tp-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - moe-a2a-backend: none - load-balance-method: round_robin - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 8192 - mamba-max-states-per-path: 3 - mamba-ssm-dtype: bfloat16 - max-mamba-cache-size: 1536 - context-length: 262144 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: triton - disable-shared-experts-fusion: true - speculative-algorithm: NEXTN - speculative-num-steps: 6 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - speculative-moe-runner-backend: flashinfer_trtllm - speculative-moe-a2a-backend: none - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - prefill-decode-interval: 0 - mem-fraction-static: 0.85 - max-running-requests: 1 - pp-max-micro-batch-size: 1 - prefill-max-requests: 1 - stream-interval: 20 - decode-log-interval: 10 - watchdog-timeout: 1000000 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-size: 32 - enable-metrics: true - enable-linear-replayssm-spec: true - disable-attn-tp-gather: true - cuda-graph-max-bs-decode: 1 - cuda-graph-bs-decode: - - 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml deleted file mode 100644 index ad5b4c86e7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml +++ /dev/null @@ -1,135 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831 -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router - enable_multiple_frontends: false -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: - /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: - /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/flashinfer-cache - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/sglang-cache - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 1 - moe-dense-tp-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - moe-a2a-backend: none - load-balance-method: round_robin - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 8192 - mamba-max-states-per-path: 3 - mamba-ssm-dtype: bfloat16 - max-mamba-cache-size: 1536 - context-length: 262144 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: triton - disable-shared-experts-fusion: true - speculative-algorithm: NEXTN - speculative-num-steps: 6 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - speculative-moe-runner-backend: flashinfer_trtllm - speculative-moe-a2a-backend: none - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - prefill-decode-interval: 0 - mem-fraction-static: 0.85 - max-running-requests: 2 - pp-max-micro-batch-size: 2 - prefill-max-requests: 2 - cuda-graph-max-bs-decode: 2 - cuda-graph-bs-decode: - - 1 - - 2 - stream-interval: 20 - decode-log-interval: 10 - watchdog-timeout: 1000000 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-size: 128 - enable-metrics: true - enable-linear-replayssm-spec: true - disable-attn-tp-gather: true -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml deleted file mode 100644 index 755cc77d0e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c48-mtp-hicache-jid2530017 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml deleted file mode 100644 index 24a388391d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c52-mtp-hicache-jid2527406 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml deleted file mode 100644 index ee3999af7b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: agg-gb300-tp2-c64-mtp-hicache-jid2527410 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -frontend: - type: sglang-router -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 2 - env: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml deleted file mode 100644 index bc672a8342..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml +++ /dev/null @@ -1,132 +0,0 @@ -schema: 2 -name: agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831 -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -frontend: - type: sglang-router - enable_multiple_frontends: false -engine: sglang -roles: - agg: - nodes: 2 - workers: 1 - gpus: 8 - env: - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/c7-mrr1/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/c7-mrr1/flashinfer-cache - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/c7-mrr1/sglang-cache - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 1 - moe-dense-tp-size: 8 - enable-dp-attention: false - enable-dp-lm-head: false - moe-a2a-backend: none - load-balance-method: round_robin - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 8192 - mamba-max-states-per-path: 3 - mamba-ssm-dtype: bfloat16 - max-mamba-cache-size: 1536 - context-length: 262144 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - linear-attn-decode-backend: triton - disable-shared-experts-fusion: true - speculative-algorithm: NEXTN - speculative-num-steps: 6 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - speculative-moe-runner-backend: flashinfer_cutedsl - speculative-moe-a2a-backend: none - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - mem-fraction-static: 0.85 - max-running-requests: 1 - pp-max-micro-batch-size: 1 - prefill-max-requests: 1 - cuda-graph-max-bs-decode: 1 - cuda-graph-bs-decode: - - 1 - disable-prefill-cuda-graph: true - stream-interval: 20 - decode-log-interval: 10 - watchdog-timeout: 1000000 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-size: 32 - enable-metrics: true - enable-linear-replayssm-spec: true - disable-attn-tp-gather: true -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - TP: '8' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..946105d17c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -0,0 +1,113 @@ +# AgentX qwen3.5 sglang gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 + slurm: + time_limit: '8:00:00' + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 2 + frontend: + type: sglang-router + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +override_c1: + name: agg-gb300-tp2-c1-mtp-hicache-jid2530006 + +override_c24: + name: agg-gb300-tp2-c24-mtp-hicache-jid2530012 + +override_c32: + name: agg-gb300-tp2-c32-mtp-hicache-jid2530013 + +override_c40: + name: agg-gb300-tp2-c40-mtp-hicache-jid2530015 + +override_c48: + name: agg-gb300-tp2-c48-mtp-hicache-jid2530017 + +override_c52: + name: agg-gb300-tp2-c52-mtp-hicache-jid2527406 + +override_c64: + name: agg-gb300-tp2-c64-mtp-hicache-jid2527410 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml deleted file mode 100644 index 3580fbbe29..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml +++ /dev/null @@ -1,211 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - max-mamba-cache-size: 320 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - mamba-max-states-per-path: 1 - decode: - nodes: 1 - workers: 1 - gpus: 2 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 80 - max-running-requests: 80 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - decode-log-interval: 10 - mamba-max-states-per-path: -1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml deleted file mode 100644 index 908f37b284..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml +++ /dev/null @@ -1,206 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - decode: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 160 - watchdog-timeout: 1000000 - decode-log-interval: 10 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml deleted file mode 100644 index a9e4f40a9a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml +++ /dev/null @@ -1,206 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - decode: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 64 - watchdog-timeout: 1000000 - decode-log-interval: 10 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml deleted file mode 100644 index 3854d0ad20..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml +++ /dev/null @@ -1,206 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - decode: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 64 - watchdog-timeout: 1000000 - decode-log-interval: 10 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml deleted file mode 100644 index d8325d5792..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml +++ /dev/null @@ -1,206 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - decode: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 80 - watchdog-timeout: 1000000 - decode-log-interval: 10 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml deleted file mode 100644 index 8a776aa9af..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml +++ /dev/null @@ -1,206 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - decode: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 64 - watchdog-timeout: 1000000 - decode-log-interval: 10 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml deleted file mode 100644 index c0ff95a0f7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml +++ /dev/null @@ -1,206 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - source: - rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - decode: - nodes: 1 - workers: 1 - gpus: 4 - env: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - decode-log-interval: 10 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml deleted file mode 100644 index e790701576..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml +++ /dev/null @@ -1,250 +0,0 @@ -schema: 2 -name: disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831 -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 -dynamo: - install: true - source: - rev: 61c37bc33c3e88da3facaa39fa2b5caf616740ba -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 1 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-replica-sync: true - router-queue-threshold: None - router-temperature: '1.0' -engine: sglang -roles: - prefill: - nodes: 3 - workers: 3 - gpus: 4 - env: - # The 0831 image contains the PP+spec runtime merged by SGLang #35758, - # but still carries the pre-merge PP+spec assert in validation_hook.py. - PYTHONOPTIMIZE: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-upstream-main/prefill-deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-upstream-main/prefill-flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '8192' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_PP_LAYER_PARTITION: 16,16,16,12 - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - SGLANG_FLASHINFER_AUTOTUNE_EXTEND: '1' - SGLANG_CACHE_DIR: /tmp/agentx-upstream-main/prefill-sglang-cache - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 1 - pipeline-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false - disable-overlap-schedule: true - enable-dynamic-chunking: false - mamba-ssm-dtype: bfloat16 - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 1048576 - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1536 - max-running-requests: 128 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - linear-attn-decode-backend: triton - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - watchdog-timeout: 1000000 - log-level: info - nccl-port: 29500 - scheduler-recv-interval: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - moe-dense-tp-size: 1 - disable-shared-experts-fusion: true - moe-a2a-backend: none - speculative-moe-a2a-backend: none - speculative-moe-runner-backend: flashinfer_trtllm - pp-async-batch-depth: 1 - disable-cuda-graph: true - decode: - nodes: 2 - workers: 2 - gpus: 4 - env: - SGLANG_USE_SYMM_MEM_DP_SYNC: 'true' - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '4' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-upstream-main/decode-deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-upstream-main/decode-flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_CACHE_DIR: /tmp/agentx-upstream-main/decode-sglang-cache - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - data-parallel-size: 4 - expert-parallel-size: 4 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: round_robin - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - linear-attn-decode-backend: triton - disable-shared-experts-fusion: true - moe-a2a-backend: flashinfer - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - speculative-draft-model-quantization: nvfp4_online - speculative-moe-runner-backend: flashinfer_trtllm_routed - speculative-moe-a2a-backend: flashinfer - chunked-prefill-size: 4096 - mem-fraction-static: 0.7 - max-mamba-cache-size: 468 - max-running-requests: 320 - cuda-graph-max-bs: 80 - disaggregation-decode-extra-slots: 2 - stream-interval: 30 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 32 - decode-log-interval: 30 - watchdog-timeout: 1000000 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml deleted file mode 100644 index 9146587211..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml +++ /dev/null @@ -1,250 +0,0 @@ -schema: 2 -name: disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831 -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 -dynamo: - install: true - source: - rev: 61c37bc33c3e88da3facaa39fa2b5caf616740ba -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: dedicated - - name: nats - type: nats - placement: - node: dedicated - options: - max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 1 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-replica-sync: true - router-queue-threshold: None - router-temperature: '1.0' -engine: sglang -roles: - prefill: - nodes: 3 - workers: 3 - gpus: 4 - env: - # The 0831 image contains the PP+spec runtime merged by SGLang #35758, - # but still carries the pre-merge PP+spec assert in validation_hook.py. - PYTHONOPTIMIZE: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '8192' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_PP_LAYER_PARTITION: 16,16,16,12 - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - SGLANG_FLASHINFER_AUTOTUNE_EXTEND: '1' - SGLANG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-sglang-cache - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 1 - pipeline-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false - disable-overlap-schedule: true - enable-dynamic-chunking: false - mamba-ssm-dtype: bfloat16 - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 1048576 - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1536 - max-running-requests: 128 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - speculative-algorithm: NEXTN - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - linear-attn-decode-backend: triton - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - watchdog-timeout: 1000000 - log-level: info - nccl-port: 29500 - scheduler-recv-interval: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - moe-dense-tp-size: 1 - disable-shared-experts-fusion: true - moe-a2a-backend: none - speculative-moe-a2a-backend: none - speculative-moe-runner-backend: flashinfer_trtllm - pp-async-batch-depth: 1 - disable-cuda-graph: true - decode: - nodes: 4 - workers: 4 - gpus: 4 - env: - SGLANG_USE_SYMM_MEM_DP_SYNC: 'true' - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '4' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/decode-deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-followup/c565-cutedsl-g3072/decode-flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/decode-sglang-cache - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - args: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - data-parallel-size: 4 - expert-parallel-size: 4 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: round_robin - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - linear-attn-decode-backend: triton - disable-shared-experts-fusion: true - moe-a2a-backend: flashinfer - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - speculative-draft-model-quantization: nvfp4_online - speculative-moe-runner-backend: flashinfer_trtllm_routed - speculative-moe-a2a-backend: flashinfer - chunked-prefill-size: 4096 - mem-fraction-static: 0.7 - max-mamba-cache-size: 468 - max-running-requests: 320 - cuda-graph-max-bs: 80 - disaggregation-decode-extra-slots: 2 - stream-interval: 30 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 32 - decode-log-interval: 30 - watchdog-timeout: 1000000 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml new file mode 100644 index 0000000000..3caef43cbc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml @@ -0,0 +1,287 @@ +# AgentX qwen3.5 sglang gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml:override_. + +schema: 2 + +base: + model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + precision: fp4 + dynamo: + install: true + source: + rev: 61c37bc33c3e88da3facaa39fa2b5caf616740ba + slurm: + time_limit: '8:00:00' + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 8 + frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-replica-sync: true + router-queue-threshold: None + router-temperature: '1.0' + engine: sglang + roles: + prefill: + nodes: 3 + workers: 3 + gpus: 4 + env: + # The 0831 image contains the PP+spec runtime merged by SGLang #35758, + # but still carries the pre-merge PP+spec assert in validation_hook.py. + PYTHONOPTIMIZE: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '8192' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_PP_LAYER_PARTITION: 16,16,16,12 + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_FLASHINFER_AUTOTUNE_EXTEND: '1' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 1 + pipeline-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + enable-symm-mem: false + disable-overlap-schedule: true + enable-dynamic-chunking: false + mamba-ssm-dtype: bfloat16 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 1048576 + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1536 + max-running-requests: 128 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + linear-attn-decode-backend: triton + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + watchdog-timeout: 1000000 + log-level: info + nccl-port: 29500 + scheduler-recv-interval: 1 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + moe-dense-tp-size: 1 + disable-shared-experts-fusion: true + moe-a2a-backend: none + speculative-moe-a2a-backend: none + speculative-moe-runner-backend: flashinfer_trtllm + pp-async-batch-depth: 1 + disable-cuda-graph: true + decode: + gpus: 4 + env: + SGLANG_USE_SYMM_MEM_DP_SYNC: 'true' + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '4' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + data-parallel-size: 4 + expert-parallel-size: 4 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: round_robin + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + moe-a2a-backend: flashinfer + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-draft-model-quantization: nvfp4_online + speculative-moe-runner-backend: flashinfer_trtllm_routed + speculative-moe-a2a-backend: flashinfer + chunked-prefill-size: 4096 + mem-fraction-static: 0.7 + max-mamba-cache-size: 468 + max-running-requests: 320 + cuda-graph-max-bs: 80 + disaggregation-decode-extra-slots: 2 + stream-interval: 30 + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 32 + decode-log-interval: 30 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +override_3p4d_c565: + name: disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831 + roles: + prefill: + env: + SGLANG_DG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-flashinfer-cache + SGLANG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-sglang-cache + args: + speculative-num-steps: 4 + speculative-num-draft-tokens: 5 + decode: + nodes: 4 + workers: 4 + env: + SGLANG_DG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/decode-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-followup/c565-cutedsl-g3072/decode-flashinfer-cache + SGLANG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/decode-sglang-cache + args: + speculative-num-steps: 4 + speculative-num-draft-tokens: 5 + +override_3p2d_c704: + name: disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831 + roles: + prefill: + env: + SGLANG_DG_CACHE_DIR: /tmp/agentx-upstream-main/prefill-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-upstream-main/prefill-flashinfer-cache + SGLANG_CACHE_DIR: /tmp/agentx-upstream-main/prefill-sglang-cache + args: + speculative-num-steps: 5 + speculative-num-draft-tokens: 6 + decode: + nodes: 2 + workers: 2 + env: + SGLANG_DG_CACHE_DIR: /tmp/agentx-upstream-main/decode-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-upstream-main/decode-flashinfer-cache + SGLANG_CACHE_DIR: /tmp/agentx-upstream-main/decode-sglang-cache + args: + speculative-num-steps: 5 + speculative-num-draft-tokens: 6 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..b38e93698f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,276 @@ +# AgentX qwen3.5 sglang gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 + dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a + slurm: + time_limit: '8:00:00' + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 8 + frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID + engine: sglang + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + watchdog-timeout: 1000000 + decode-log-interval: 10 + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +override_tp4_tp4_c8: + name: disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030 + roles: + decode: + args: + cuda-graph-max-bs: 64 + +override_tp4_tp4_c16: + name: disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027 + roles: + decode: + args: + cuda-graph-max-bs: 64 + +override_tp4_tp4_c32: + name: disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028 + roles: + decode: + args: + cuda-graph-max-bs: 64 + +override_tp4_tp4_c64: + name: disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029 + roles: + decode: + args: + cuda-graph-max-bs: 80 + +override_tp4_tp4_c96: + name: disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409 + roles: + decode: + args: + cuda-graph-max-bs: 128 + +override_tp4_tp4_c128: + name: disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417 + roles: + decode: + args: + cuda-graph-max-bs: 160 + +override_tp2_tp2_c72: + name: disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415 + resources: + gpus_per_node: 2 + roles: + prefill: + gpus: 2 + args: + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + tensor-parallel-size: 2 + max-mamba-cache-size: 320 + mamba-max-states-per-path: 1 + decode: + gpus: 2 + args: + tensor-parallel-size: 2 + max-mamba-cache-size: 80 + max-running-requests: 80 + cuda-graph-max-bs: 128 + mamba-max-states-per-path: -1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml deleted file mode 100644 index f498c1d931..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml +++ /dev/null @@ -1,222 +0,0 @@ -schema: 2 -name: disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 1 - env: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 8192 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 1 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - nodes: 1 - workers: 1 - gpus: 2 - env: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - enable_padding: true - enable_attention_dp: false - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.85 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 8 - max_num_tokens: 56 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 2 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 6 - stream_interval: 20 - tensor_parallel_size: 2 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml deleted file mode 100644 index 076f17ae91..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml +++ /dev/null @@ -1,219 +0,0 @@ -schema: 2 -name: disagg-gb300-1p7d-dep4-tep8-c7-b1-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 8192 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 4 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - nodes: 14 - workers: 7 - gpus: 8 - env: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - enable_attention_dp: false - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.85 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 1 - max_num_tokens: 8 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 7 - stream_interval: 20 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml deleted file mode 100644 index 091438994f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml +++ /dev/null @@ -1,221 +0,0 @@ -schema: 2 -name: disagg-gb300-2p2d-dep1-tep2-c52-b4-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 1 - workers: 2 - gpus: 1 - env: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 8192 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 1 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - nodes: 1 - workers: 2 - gpus: 2 - env: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - enable_attention_dp: false - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.85 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 4 - max_num_tokens: 28 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 2 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 6 - stream_interval: 20 - tensor_parallel_size: 2 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml deleted file mode 100644 index f3134246e6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml +++ /dev/null @@ -1,236 +0,0 @@ -schema: 2 -name: disagg-gb300-2p3d-tep2-tep8-c96-b128-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 1 - workers: 2 - gpus: 2 - env: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: false - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 16384 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 2 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 2 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - - 16384 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - nodes: 6 - workers: 3 - gpus: 8 - env: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - enable_padding: true - enable_attention_dp: false - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.85 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 128 - max_num_tokens: 896 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 6 - stream_interval: 20 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml deleted file mode 100644 index cddc0d8a81..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml +++ /dev/null @@ -1,220 +0,0 @@ -schema: 2 -name: disagg-gb300-3p1d-dep4-dep16-c565-b8-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 3 - workers: 3 - gpus: 4 - env: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 8192 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 4 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - nodes: 4 - workers: 1 - gpus: 16 - env: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - enable_padding: true - enable_attention_dp: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.85 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 8 - max_num_tokens: 56 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 6 - stream_interval: 20 - tensor_parallel_size: 16 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml deleted file mode 100644 index 3c0b23c7b6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml +++ /dev/null @@ -1,223 +0,0 @@ -schema: 2 -name: disagg-gb300-3p2d-dep4-dep4-c704-b32-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - source: - git: https://github.com/cquil11/dynamo.git - rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 -engine: - type: trtllm - publish_events_and_metrics: false -roles: - prefill: - nodes: 3 - workers: 3 - gpus: 4 - env: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 8192 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 4 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - nodes: 2 - workers: 2 - gpus: 4 - env: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - args: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - enable_padding: true - enable_attention_dp: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 32 - max_num_tokens: 224 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 6 - stream_interval: 20 - tensor_parallel_size: 4 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..d607b6abae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,408 @@ +# AgentX qwen3.5 trtllm gb300-fp4 recipes: shared settings in base, one +# override per benchmark configuration. Select one with +# CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_. + +schema: 2 + +base: + model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + slurm: + time_limit: 04:00:00 + health_check: + max_attempts: 540 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + engine: + type: trtllm + publish_events_and_metrics: false + roles: + prefill: + nodes: 1 + env: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: true + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + env: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: true + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + trust_remote_code: true + frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 + +override_1p7d_dep4_tep8_c7_b1: + name: disagg-gb300-1p7d-dep4-tep8-c7-b1-mtp-kvoffload + roles: + prefill: + workers: 1 + gpus: 4 + args: + moe_expert_parallel_size: 4 + tensor_parallel_size: 4 + decode: + nodes: 14 + workers: 7 + gpus: 8 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + max_batch_size: 1 + max_num_tokens: 8 + moe_expert_parallel_size: 8 + speculative_config: + max_draft_len: 7 + tensor_parallel_size: 8 + +override_1p1d_dep1_tep2_c44_b8: + name: disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload + roles: + prefill: + workers: 1 + gpus: 1 + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + args: + moe_expert_parallel_size: 1 + tensor_parallel_size: 1 + decode: + nodes: 1 + workers: 1 + gpus: 2 + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + max_batch_size: 8 + max_num_tokens: 56 + moe_expert_parallel_size: 2 + tensor_parallel_size: 2 + +override_2p2d_dep1_tep2_c52_b4: + name: disagg-gb300-2p2d-dep1-tep2-c52-b4-mtp-kvoffload + roles: + prefill: + workers: 2 + gpus: 1 + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + args: + moe_expert_parallel_size: 1 + tensor_parallel_size: 1 + decode: + nodes: 1 + workers: 2 + gpus: 2 + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + max_batch_size: 4 + max_num_tokens: 28 + moe_expert_parallel_size: 2 + tensor_parallel_size: 2 + +override_2p3d_tep2_tep8_c96_b128: + name: disagg-gb300-2p3d-tep2-tep8-c96-b128-mtp-kvoffload + roles: + prefill: + workers: 2 + gpus: 2 + args: + enable_attention_dp: false + max_num_tokens: 16384 + moe_expert_parallel_size: 2 + tensor_parallel_size: 2 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + - 16384 + decode: + nodes: 6 + workers: 3 + gpus: 8 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + max_batch_size: 128 + max_num_tokens: 896 + moe_expert_parallel_size: 8 + tensor_parallel_size: 8 + +override_3p1d_dep4_dep16_c565_b8: + name: disagg-gb300-3p1d-dep4-dep16-c565-b8-mtp-kvoffload + roles: + prefill: + nodes: 3 + workers: 3 + gpus: 4 + args: + moe_expert_parallel_size: 4 + tensor_parallel_size: 4 + decode: + nodes: 4 + workers: 1 + gpus: 16 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + max_batch_size: 8 + max_num_tokens: 56 + moe_expert_parallel_size: 16 + tensor_parallel_size: 16 + +override_3p2d_dep4_dep4_c704_b32: + name: disagg-gb300-3p2d-dep4-dep4-c704-b32-mtp-kvoffload + roles: + prefill: + nodes: 3 + workers: 3 + gpus: 4 + args: + moe_expert_parallel_size: 4 + tensor_parallel_size: 4 + decode: + nodes: 2 + workers: 2 + gpus: 4 + args: + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + free_gpu_memory_fraction: 0.8 + max_batch_size: 32 + max_num_tokens: 224 + moe_expert_parallel_size: 4 + tensor_parallel_size: 4 diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh index ce28224f88..954007c8f0 100755 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh @@ -103,7 +103,6 @@ install_agentic_deps OFFLOAD_ARGS=() if require_agentic_kv_offload_backend vllm-simple; then - python3 "$(dirname "$0")/../../../runners/patch_vllm_simple_kv_offload.py" CPU_OFFLOAD_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024)) export VLLM_USE_SIMPLE_KV_OFFLOAD=1 OFFLOAD_CONFIG=$(printf \ diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh index c606792017..508c0d0359 100755 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh @@ -40,7 +40,6 @@ install_agentic_deps OFFLOAD_ARGS=() if require_agentic_kv_offload_backend vllm-simple; then - python3 "$(dirname "$0")/../../../runners/patch_vllm_simple_kv_offload.py" CPU_OFFLOAD_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024)) export VLLM_USE_SIMPLE_KV_OFFLOAD=1 OFFLOAD_CONFIG=$(printf \ diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..49bdd63b51 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,227 @@ +# DeepSeek-V4-Pro-0813 AgentX on MI355X with ATOM DSpark K6 (six draft tokens, +# a verify window of 7). Low concurrency is TP-only; the wide band runs DP +# attention with native RCCL expert parallelism. The KV cache is GPU-resident. +base: + schema: 2 + name: dsv4-fp4-mi355x-atom-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 + container: rocm/atom-dev:nightly_202609161445 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: atomesh + container_image: rocm/atom-dev@sha256:f00a9dc588a380038fdad54cd57a956043cb316ddda54ebb1b62a24f8caa343b + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: atom + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + timeout-keep-alive: 900 + data-parallel-size: 1 + index-cache-dtype: fp4 + enable-prefix-caching: true + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + attn-prefill-chunk-size: 16384 + state-checkpoint-interval-tokens: 8192 + level: 3 + cudagraph-mode: FULL + method: dspark + num-speculative-tokens: 6 + env: + AITER_BF16_FP8_MOE_BOUND: '0' + AITER_LOG_LEVEL: WARNING + ATOM_MOE_GU_ITLV: '1' + ATOM_DISABLE_MMAP: 'true' + ATOM_DEBUG_PREFIX_HITS: '1' + ATOM_PROFILER_MORE: '0' + ATOM_PROFILER_TIMEOUT: '1200' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + # Long AgentX stalls exceed AIPerf's 30 s TCP_USER_TIMEOUT default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' + +# One variant per point. Admission is 2x CONC. KV is BF16 through concurrency 16 +# and FP8 in the DEP band. DEP pins sessions to one DP-attention rank so later +# turns reach their prefix KV, and captures every small per-rank decode batch. +override_tp8_c1: + roles: + agg: + args: + max-num-seqs: 2 + kv-cache-dtype: bf16 + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + args: + max-num-seqs: 4 + kv-cache-dtype: bf16 + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + args: + max-num-seqs: 8 + kv-cache-dtype: bf16 + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + args: + max-num-seqs: 16 + kv-cache-dtype: bf16 + benchmark: + env: + CONC: '8' + +override_tp8_c16: + roles: + agg: + args: + max-num-seqs: 32 + kv-cache-dtype: bf16 + benchmark: + env: + CONC: '16' + +override_tp8_c48_dep: + roles: + agg: + args: + max-num-seqs: 96 + kv-cache-dtype: fp8 + enable-expert-parallel: true + enable-dp-attention: true + all2all-backend: rccl + dp-load-balance: least_tokens + moe-backend: standard + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]' + env: + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '10' + benchmark: + env: + CONC: '48' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c64_dep: + roles: + agg: + args: + max-num-seqs: 128 + kv-cache-dtype: fp8 + enable-expert-parallel: true + enable-dp-attention: true + all2all-backend: rccl + dp-load-balance: least_tokens + moe-backend: standard + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]' + env: + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '10' + benchmark: + env: + CONC: '64' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c96_dep: + roles: + agg: + args: + max-num-seqs: 192 + kv-cache-dtype: fp8 + enable-expert-parallel: true + enable-dp-attention: true + all2all-backend: rccl + dp-load-balance: least_tokens + moe-backend: standard + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]' + env: + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '10' + benchmark: + env: + CONC: '96' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c128_dep: + roles: + agg: + args: + max-num-seqs: 256 + kv-cache-dtype: fp8 + enable-expert-parallel: true + enable-dp-attention: true + all2all-backend: rccl + dp-load-balance: least_tokens + moe-backend: standard + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]' + env: + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '10' + benchmark: + env: + CONC: '128' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c256_dep: + roles: + agg: + args: + max-num-seqs: 512 + kv-cache-dtype: fp8 + enable-expert-parallel: true + enable-dp-attention: true + all2all-backend: rccl + dp-load-balance: least_tokens + moe-backend: standard + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]' + env: + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '10' + benchmark: + env: + CONC: '256' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..36bba702cb --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,408 @@ +# DeepSeek-V4-Pro-0813 FP4 AgentX on B200 with SGLang DSpark (block 6). TP8 +# covers the low-concurrency points (GPU KV, then HiCache); DEP8 attention-DP +# fronted by the SGLang Model Gateway covers the high-concurrency HiCache points. +base: + schema: 2 + name: dsv4-fp4-b200-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 + container: lmsysorg/sglang:v0.5.19-cu130 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tensor-parallel-size: 8 + enable-deepseek-v4-fp4-indexer: true + disable-flashinfer-autotune: true + tool-call-parser: deepseekv4 + reasoning-parser: deepseek-v4 + chat-template: /infmax-workspace/benchmarks/single_node/chat_templates/deepseek_v4_thinking.jinja + watchdog-timeout: 1800 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + # The checkpoint lives on Lustre: prefetch sequentially across local + # ranks so post-load repacking reads from page cache. + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + enable-metrics: true + enable-cache-report: true + env: + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + # Outlast AIPerf's pooled keep-alive connections. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_USE_JIT_NORM: '1' + SGLANG_OPT_USE_JIT_INDEXER_METADATA: '1' + SGLANG_OPT_USE_TOPK_V2: '1' + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + # The image's first ptxas on the legacy search path. + TRITON_PTXAS_PATH: /usr/local/cuda-13.0/bin/ptxas + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point; admission is 2x CONC. TP8 captures graphs up to the +# admission cap and runs the FlashInfer MXFP4 MoE; its replicated HiCache +# pools fit ratio 2.75 (~121 GiB/rank) on 1.7 TiB hosts. DEP8 shards the host +# pools (ratio 8), caps graphs at 32, runs MegaMoE with 6144 prefill tokens per +# DP rank, and balances DP admission; the gateway routes sessions by AIPerf +# correlation ID. C160 interleaves prefill more often and bounds the gateway's +# load imbalance. +override_tp8_c1: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 2 + cuda-graph-max-bs: 2 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c2: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 4 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + +override_tp8_c3: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 6 + cuda-graph-max-bs: 6 + benchmark: + env: + CONC: '3' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 8 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c5: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 10 + cuda-graph-max-bs: 10 + benchmark: + env: + CONC: '5' + KV_OFFLOADING: none + +override_tp8_hicache_c8: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 16 + cuda-graph-max-bs: 16 + enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 2.75 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_hicache_c10: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 20 + cuda-graph-max-bs: 20 + enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 2.75 + benchmark: + env: + CONC: '10' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_hicache_c16: + roles: + agg: + args: + moe-runner-backend: flashinfer_mxfp4 + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + chunked-prefill-size: 8192 + max-running-requests: 32 + cuda-graph-max-bs: 32 + enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 2.75 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_dep8_hicache_c64: + frontend: + type: sglang-router + args: + policy: cache_aware + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + incremental-streaming-output: true + stream-interval: 20 + tokenizer-worker-num: 8 + load-balance-method: total_requests + load-snapshot-publish-interval: 1 + prefill-decode-interval: 24 + moe-a2a-backend: megamoe + enable-w4a4-mxfp4-megamoe: true + disable-shared-experts-fusion: true + mem-fraction-static: 0.88 + swa-full-tokens-ratio: 0.02 + chunked-prefill-size: 49152 + max-running-requests: 128 + cuda-graph-max-bs: 32 + enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 8 + env: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320' + benchmark: + env: + CONC: '64' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + +override_dep8_hicache_c96: + frontend: + type: sglang-router + args: + policy: cache_aware + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + incremental-streaming-output: true + stream-interval: 20 + tokenizer-worker-num: 8 + load-balance-method: total_requests + load-snapshot-publish-interval: 1 + prefill-decode-interval: 24 + moe-a2a-backend: megamoe + enable-w4a4-mxfp4-megamoe: true + disable-shared-experts-fusion: true + mem-fraction-static: 0.88 + swa-full-tokens-ratio: 0.02 + chunked-prefill-size: 49152 + max-running-requests: 192 + cuda-graph-max-bs: 32 + enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 8 + env: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320' + benchmark: + env: + CONC: '96' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + +override_dep8_hicache_c128: + frontend: + type: sglang-router + args: + policy: cache_aware + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + incremental-streaming-output: true + stream-interval: 20 + tokenizer-worker-num: 8 + load-balance-method: total_requests + load-snapshot-publish-interval: 1 + prefill-decode-interval: 24 + moe-a2a-backend: megamoe + enable-w4a4-mxfp4-megamoe: true + disable-shared-experts-fusion: true + mem-fraction-static: 0.88 + swa-full-tokens-ratio: 0.02 + chunked-prefill-size: 49152 + max-running-requests: 256 + cuda-graph-max-bs: 32 + enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 8 + env: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320' + benchmark: + env: + CONC: '128' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + +override_dep8_hicache_c160: + frontend: + type: sglang-router + args: + policy: cache_aware + balance-abs-threshold: 32 + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + incremental-streaming-output: true + stream-interval: 20 + tokenizer-worker-num: 8 + load-balance-method: total_requests + load-snapshot-publish-interval: 1 + prefill-decode-interval: 20 + moe-a2a-backend: megamoe + enable-w4a4-mxfp4-megamoe: true + disable-shared-experts-fusion: true + mem-fraction-static: 0.88 + swa-full-tokens-ratio: 0.02 + chunked-prefill-size: 49152 + max-running-requests: 320 + cuda-graph-max-bs: 32 + enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 8 + env: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320' + benchmark: + env: + CONC: '160' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..3beb2ef09c --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,274 @@ +# DeepSeek-V4-Pro-0813 AgentX on B300 with SGLang DSpark (block 6). Two arms: +# TP8 with the flashinfer MXFP4 MoE and GPU-resident KV, and DEP8 (attention +# DP + Mega-MoE + FP4 indexer) behind the SGLang router with a HiCache DRAM tier. +base: + schema: 2 + name: dsv4-fp4-b300-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 + container: lmsysorg/sglang:nightly-dev-20260901-07c8f729 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tensor-parallel-size: 8 + moe-runner-backend: flashinfer_mxfp4 + disable-flashinfer-autotune: true + mem-fraction-static: 0.88 + swa-full-tokens-ratio: 0.1 + allow-auto-truncate: true + chunked-prefill-size: 8192 + tool-call-parser: deepseekv4 + reasoning-parser: deepseek-v4 + chat-template: /infmax-workspace/benchmarks/single_node/chat_templates/deepseek_v4_thinking.jinja + watchdog-timeout: 1800 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + attention-backend: compressed + page-size: 256 + disable-shared-experts-fusion: true + enable-metrics: true + enable-cache-report: true + env: + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: '1' + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_USE_JIT_NORM: '1' + SGLANG_OPT_USE_JIT_INDEXER_METADATA: '1' + SGLANG_OPT_USE_TOPK_V2: '1' + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + # Triton compiles with the image's CUDA ptxas. + TRITON_PTXAS_PATH: /usr/local/cuda/bin/ptxas + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. TP8: admission is 2x CONC and the graph batch 4x +# CONC, capped at 64. +override_tp8_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp8_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp8_c32: + roles: + agg: + args: + max-running-requests: 64 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: none + +# DEP8 replaces the TP8 MoE path: the router keeps each session on the DP rank +# holding its radix prefix, the global prefill chunk is 8192 per rank, decode +# graphs cover the padded speculative batch across ranks, and AIPerf owns the +# warmup. HiCache capacity is a host/device ratio; 3 keeps the tier near 2 TB. +# Mega-MoE's transient workspace sits outside the static pool, so the static +# fraction steps down with CONC: 0.90 to c256, 0.88 at c384, 0.86 from c512. +override_dep8_c32: + frontend: &dep8_frontend + type: sglang-router + args: + policy: consistent_hashing + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + # A transient router-to-engine send failure would otherwise abort the run. + retry-max-retries: 8 + retry-initial-backoff-ms: 500 + retry-max-backoff-ms: 10000 + retry-backoff-multiplier: 2 + roles: + agg: + args: + <<: &dep8_args + moe-runner-backend: null + data-parallel-size: 8 + tokenizer-worker-num: 8 + enable-prefill-delayer: true + prefill-decode-interval: 20 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + incremental-streaming-output: true + stream-interval: 20 + expert-parallel-size: 8 + moe-a2a-backend: megamoe + enable-w4a4-mxfp4-megamoe: true + enable-deepseek-v4-fp4-indexer: true + chunked-prefill-size: 65536 + cuda-graph-max-bs-decode: 544 + swa-full-tokens-ratio: 0.075 + enable-hierarchical-cache: true + hicache-ratio: 3 + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + skip-server-warmup: true + max-running-requests: 64 + mem-fraction-static: 0.9 + env: &dep8_env + # Covers the 8192-token per-rank prefill budget. + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320' + benchmark: + env: + <<: &dep8_client + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2849' + CONC: '32' +override_dep8_c64: + frontend: *dep8_frontend + roles: + agg: + args: + <<: *dep8_args + max-running-requests: 128 + mem-fraction-static: 0.9 + env: *dep8_env + benchmark: + env: + <<: *dep8_client + CONC: '64' + +override_dep8_c128: + frontend: *dep8_frontend + roles: + agg: + args: + <<: *dep8_args + max-running-requests: 256 + mem-fraction-static: 0.9 + env: *dep8_env + benchmark: + env: + <<: *dep8_client + CONC: '128' + +override_dep8_c256: + frontend: *dep8_frontend + roles: + agg: + args: + <<: *dep8_args + max-running-requests: 512 + mem-fraction-static: 0.9 + env: *dep8_env + benchmark: + env: + <<: *dep8_client + CONC: '256' + +override_dep8_c384: + frontend: *dep8_frontend + roles: + agg: + args: + <<: *dep8_args + max-running-requests: 768 + mem-fraction-static: 0.88 + env: *dep8_env + benchmark: + env: + <<: *dep8_client + CONC: '384' + +override_dep8_c512: + frontend: *dep8_frontend + roles: + agg: + args: + <<: *dep8_args + max-running-requests: 1024 + mem-fraction-static: 0.86 + env: *dep8_env + benchmark: + env: + <<: *dep8_client + CONC: '512' + +override_dep8_c576: + frontend: *dep8_frontend + roles: + agg: + args: + <<: *dep8_args + max-running-requests: 1152 + mem-fraction-static: 0.86 + env: *dep8_env + benchmark: + env: + <<: *dep8_client + CONC: '576' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..b3aacda995 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,242 @@ +# DeepSeek-V4-Pro-0813 AgentX on MI355X with SGLang DSpark. Low concurrency +# keeps KV on GPU; higher concurrency extends it to host DRAM through HiCache, +# and the widest band runs DP attention behind a cache-aware SGLang router. +base: + schema: 2 + name: dsv4-fp4-mi355x-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 + container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260914 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + attention-backend: dsv4 + enable-deepseek-v4-fp4-indexer: true + page-size: 256 + swa-full-tokens-ratio: 0.1 + kv-cache-dtype: fp8_e4m3 + enforce-shared-experts-fusion: true + # No chat-template override: deepseek_v4_thinking.jinja drops tool + # definitions and tool messages. + tool-call-parser: deepseekv4 + reasoning-parser: deepseek-v4 + chunked-prefill-size: 16384 + mem-fraction-static: 0.86 + prefill-decode-interval: 20 + # DSpark is the checkpoint's bundled draft; gamma 6 is AL-optimal on the + # golden curve and verifies a window of 7. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_USE_ROCM700A: '0' + SGLANG_HACK_FLASHMLA_BACKEND: unified_kv_triton + AITER_BF16_FP8_MOE_BOUND: '0' + TORCH_BLAS_PREFER_HIPBLASLT: '1' + HSA_NO_SCRATCH_RECLAIM: '0' + # aiter batched GEMM for the absorbed MLA projections. + SGLANG_OPT_USE_AITER_BATCHED_GEMM: '1' + # Unified radix tree that releases out-of-window SWA slots, so cached + # sessions keep their trailing window under multi-turn load. + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: '1' + GPU_MAX_HW_QUEUES: '2' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC; the decode graph batch follows it +# up to 128. HiCache pins 1.5x the device KV pool per rank. DP attention keeps +# 8192 prefill tokens per rank, and the router pins each AIPerf session to the +# rank holding its prefix. Saturation points get a longer warmup drain. +override_tp8_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp8_c32_hicache: + roles: + agg: + args: + max-running-requests: 64 + cuda-graph-max-bs-decode: 64 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c48_hicache: + roles: + agg: + args: + max-running-requests: 96 + cuda-graph-max-bs-decode: 96 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + benchmark: + env: + CONC: '48' + KV_OFFLOADING: dram + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c128_dp_hicache: + frontend: + type: sglang-router + args: + policy: cache_aware + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + max-running-requests: 256 + cuda-graph-max-bs-decode: 128 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-prefill-delayer: true + enable-dp-attention-local-control-broadcast: true + tokenizer-worker-num: 8 + stream-interval: 20 + prefill-delayer-token-usage-low-watermark: 0.7 + chunked-prefill-size: 65536 + mem-fraction-static: 0.92 + env: + SGLANG_SHARED_EXPERT_TP1: '1' + SGLANG_DP_SHARED_EXPERT_LOCAL: '1' + SGLANG_DP_USE_GATHERV: '1' + SGLANG_DP_USE_REDUCE_SCATTER: '1' + GPU_MAX_HW_QUEUES: '5' + benchmark: + env: + CONC: '128' + KV_OFFLOADING: dram + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c256_dp_hicache: + frontend: + type: sglang-router + args: + policy: cache_aware + balance-abs-threshold: 32 + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + max-running-requests: 512 + cuda-graph-max-bs-decode: 128 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + enable-prefill-delayer: true + enable-dp-attention-local-control-broadcast: true + tokenizer-worker-num: 8 + stream-interval: 20 + prefill-delayer-token-usage-low-watermark: 0.7 + chunked-prefill-size: 65536 + mem-fraction-static: 0.92 + env: + SGLANG_SHARED_EXPERT_TP1: '1' + SGLANG_DP_SHARED_EXPERT_LOCAL: '1' + SGLANG_DP_USE_GATHERV: '1' + SGLANG_DP_USE_REDUCE_SCATTER: '1' + GPU_MAX_HW_QUEUES: '5' + benchmark: + env: + CONC: '256' + KV_OFFLOADING: dram + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..44c6285845 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,386 @@ +# DeepSeek-V4-Pro-0813 AgentX on B200 with vLLM native DSpark (six draft +# tokens). TP8 keeps the KV cache GPU-resident; DEP8 offloads it to host DRAM +# through SimpleCPUOffloadConnector. +base: + schema: 2 + name: dsv4-fp4-b200-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-591bb95 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 720 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + kv-cache-dtype: fp8 + block-size: 256 + max-model-len: 1048576 + gpu-memory-utilization: 0.90 + numa-bind: true + enable-cumem-allocator: true + no-enable-flashinfer-autotune: true + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v4 + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"indexer_kv_dtype":"mxfp4"}' + # Throughput runs switch to synthetic rejection at the golden acceptance length. + speculative-config: '{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"probabilistic"}' + no-disable-hybrid-kv-cache-manager: true + disable-uvicorn-access-log: true + env: + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '32768' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_DSV4_MEGA_FP8_COMBINE: '1' + VLLM_RPC_TIMEOUT: '600000' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + VLLM_FLOAT32_MATMUL_PRECISION: high + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Graphs capture every decode batch of up to +# max-num-seqs sequences in tokens (1 target + 6 drafts each). TP8 admits CONC +# with a fixed 50 GiB KV cache, and also captures piecewise mixed batches of +# 100-500 tokens. DEP8 runs one data-parallel rank per GPU behind a +# consistent-hash vLLM Router (turns of one conversation share a rank), admits +# 2x CONC across the ranks with MegaMoE experts, reserves verification slots +# beyond an 8192-token prefill budget, and splits TOTAL_CPU_DRAM_GB GB of eager +# offload across the ranks; c160 and c192 shrink the per-rank KV cache. +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + max-num-batched-tokens: 16384 + kv-cache-memory-bytes: 53687091200 + max-num-seqs: 1 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,100,200,300,400,500]}' + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + max-num-batched-tokens: 16384 + kv-cache-memory-bytes: 53687091200 + max-num-seqs: 4 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,100,200,300,400,500]}' + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c6: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + max-num-batched-tokens: 16384 + kv-cache-memory-bytes: 53687091200 + max-num-seqs: 6 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,100,200,300,400,500]}' + benchmark: + env: + CONC: '6' + KV_OFFLOADING: none + +override_tp8_c10: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + max-num-batched-tokens: 16384 + kv-cache-memory-bytes: 53687091200 + max-num-seqs: 10 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,100,200,300,400,500]}' + benchmark: + env: + CONC: '10' + KV_OFFLOADING: none + +override_tp8_c14: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + max-num-batched-tokens: 16384 + kv-cache-memory-bytes: 53687091200 + max-num-seqs: 14 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,100,200,300,400,500]}' + benchmark: + env: + CONC: '14' + KV_OFFLOADING: none + +override_tp8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + max-num-batched-tokens: 16384 + kv-cache-memory-bytes: 53687091200 + max-num-seqs: 16 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,100,105,112,200,300,400,500]}' + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_dep8_c32: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 16 + long-prefill-token-threshold: 512 + kv-cache-memory-bytes: 37580963840 + max-num-batched-tokens: 8248 + max-num-seqs: 8 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1947' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c64: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 16 + long-prefill-token-threshold: 512 + kv-cache-memory-bytes: 37580963840 + max-num-batched-tokens: 8304 + max-num-seqs: 16 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '64' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1947' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c96: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 16 + long-prefill-token-threshold: 512 + kv-cache-memory-bytes: 37580963840 + max-num-batched-tokens: 8360 + max-num-seqs: 24 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '96' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1947' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c128: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 16 + long-prefill-token-threshold: 512 + kv-cache-memory-bytes: 37580963840 + max-num-batched-tokens: 8416 + max-num-seqs: 32 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '128' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1947' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c160: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 16 + long-prefill-token-threshold: 512 + kv-cache-memory-bytes: 36507222016 + max-num-batched-tokens: 8472 + max-num-seqs: 40 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224,231,238,245,252,259,266,273,280],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '160' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1947' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c192: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 16 + long-prefill-token-threshold: 512 + kv-cache-memory-bytes: 36507222016 + max-num-batched-tokens: 8528 + max-num-seqs: 48 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224,231,238,245,252,259,266,273,280,287,294,301,308,315,322,329,336],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":243375000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '192' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1947' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..800c798e7b --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,478 @@ +# DeepSeek-V4-Pro AgentX on B300 with vLLM native MTP (three draft tokens). +# TP8 and TP4 c8 are GPU-resident; TP4 c16, DEP4 and DEP8 offload KV to host +# DRAM through SimpleCPUOffloadConnector. +base: + schema: 2 + name: dsv4-fp4-b300-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4-Pro + container: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-426e59f + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + trust-remote-code: true + no-enable-flashinfer-autotune: true + no-disable-hybrid-kv-cache-manager: true + kv-cache-dtype: fp8 + block-size: 256 + max-model-len: 1048576 + attention-config: '{"use_fp4_indexer_cache":true,"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}' + # Throughput runs switch to synthetic rejection at the golden acceptance length. + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + disable-uvicorn-access-log: true + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v4 + env: + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '32768' + VLLM_DSV4_MEGA_FP8_COMBINE: '1' + NCCL_NVLS_ENABLE: '1' + VLLM_USE_RUST_FRONTEND: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + VLLM_FLOAT32_MATMUL_PRECISION: high + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Decode graphs capture every batch of up to +# max-num-seqs sequences in tokens (1 target + 3 drafts each). TP points admit +# 2x CONC with the FlashInfer all-reduce; DEP points run one data-parallel rank +# per GPU behind a consistent-hash vLLM Router (turns of one conversation share +# a rank), admit 2x CONC across the ranks with MegaMoE experts, and keep eager +# offload so block hashes agree across ranks. DEP8 takes a larger prefill +# budget and more memory headroom. DRAM points split TOTAL_CPU_DRAM_GB GB +# across the ranks. +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 2 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8],"mode":0}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 8 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32],"mode":0}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 2 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8],"mode":0}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 4 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16],"mode":0}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 8 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32],"mode":0}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp4_c6: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 12 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48],"mode":0}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + benchmark: + env: + CONC: '6' + KV_OFFLOADING: none + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 16 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + data-parallel-size: 1 + disable-custom-all-reduce: true + gpu-memory-utilization: 0.95 + max-num-seqs: 32 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356000000000,"enable_cross_layers_blocks":"true","lazy_offload":true}}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_FLASHINFER_ALLREDUCE_BACKEND: 'auto' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '16' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1424' + +override_dep4_c48: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 1 + data-parallel-size: 4 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 512 + max-num-batched-tokens: 8192 + gpu-memory-utilization: 0.95 + max-num-seqs: 24 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356000000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '48' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1424' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep4_c64: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 1 + data-parallel-size: 4 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 512 + max-num-batched-tokens: 8192 + gpu-memory-utilization: 0.95 + max-num-seqs: 32 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356000000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '64' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1424' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c128: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 512 + max-num-batched-tokens: 16384 + gpu-memory-utilization: 0.92 + max-num-seqs: 32 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '128' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2849' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c256: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 512 + max-num-batched-tokens: 16384 + gpu-memory-utilization: 0.92 + max-num-seqs: 64 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '256' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2849' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c384: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 512 + max-num-batched-tokens: 16384 + gpu-memory-utilization: 0.92 + max-num-seqs: 96 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,292,296,300,304,308,312,316,320,324,328,332,336,340,344,348,352,356,360,364,368,372,376,380,384],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '384' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2849' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c512: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 512 + max-num-batched-tokens: 16384 + gpu-memory-utilization: 0.92 + max-num-seqs: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,292,296,300,304,308,312,316,320,324,328,332,336,340,344,348,352,356,360,364,368,372,376,380,384,388,392,396,400,404,408,412,416,420,424,428,432,436,440,444,448,452,456,460,464,468,472,476,480,484,488,492,496,500,504,508,512],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '512' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2849' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c576: + frontend: + type: vllm-router + args: + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 + disable-retries: true + setup_script: vllm-router-0.1.14.sh + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_amxf4_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 512 + max-num-batched-tokens: 16384 + gpu-memory-utilization: 0.92 + max-num-seqs: 144 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,256,260,264,268,272,276,280,284,288,292,296,300,304,308,312,316,320,324,328,332,336,340,344,348,352,356,360,364,368,372,376,380,384,388,392,396,400,404,408,412,416,420,424,428,432,436,440,444,448,452,456,460,464,468,472,476,480,484,488,492,496,500,504,508,512,516,520,524,528,532,536,540,544,548,552,556,560,564,568,572,576],"mode":0}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":356125000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + env: + PYTORCH_ALLOC_CONF: 'expandable_segments:True' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '576' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2849' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..47d2f7616e --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,231 @@ +# DeepSeek-V4-Pro-0813 AgentX on MI355X with vLLM DSpark K6 +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro?hardware=mi355x). Pure +# TP8 through c64; the wide band runs DP attention (DEP8) behind the vLLM +# Router. The KV cache is GPU-resident. +base: + schema: 2 + name: dsv4-fp4-mi355x-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 + container: vllm/vllm-openai-rocm:nightly-e9757321527ca1ecd514c07c1418dd2c53da3d19 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # 805 GiB checkpoint; cold loads can take the full engine ready timeout. + health_check: + interval_seconds: 10 + max_attempts: 1080 + # The nightly ROCm image lacks some runtime packages; see SETUP_PIP_PACKAGES. + setup_script: pip-runtime-deps.sh + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + # The DSpark checkpoint revision the legacy preflight pinned. + revision: 72e1d3230f6c080a530b0a1d46f8eb4602340597 + tokenizer-revision: 72e1d3230f6c080a530b0a1d46f8eb4602340597 + trust-remote-code: true + async-scheduling: true + distributed-executor-backend: mp + kv-cache-dtype: fp8 + max-num-batched-tokens: 8192 + tensor-parallel-size: 8 + data-parallel-size: 1 + gpu-memory-utilization: 0.86 + moe-backend: aiter + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' + # Six-token DSpark with probabilistic drafting. Throughput runs replace + # verification with the golden acceptance length. + speculative-config: '{"method": "dspark", "num_speculative_tokens": 6, "draft_sample_method": "probabilistic"}' + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + reasoning-parser: deepseek_v4 + enable-auto-tool-choice: true + enable-prefix-caching: true + no-disable-hybrid-kv-cache-manager: true + env: + SETUP_PIP_PACKAGES: Pillow fastapi uvicorn + VLLM_ENGINE_READY_TIMEOUT_S: '10800' + # Sparse SWA prefix-cache tail retention validated for this workload. + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '32768' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_ROCM_USE_AITER: '1' + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 + VLLM_ROCM_USE_AITER_MOE: '1' + # Packed MXFP4 routed experts beside a full-width FP8 shared expert fail + # to load through the fused path. + VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: '0' + # vLLM only clamps torch threads after weight loading. + OMP_NUM_THREADS: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Pure TP admits 2x CONC. DEP8 admits CONC per rank, +# interleaves prefill every 8 steps with a 16k long-prefill threshold, and +# routes each conversation to one rank: the Router hashes X-Session-ID, which +# AIPerf aliases from its stable X-Correlation-ID. +override_tp8_c1: + roles: + agg: + args: + max-num-seqs: 2 + benchmark: + env: + CONC: '1' + +override_tp8_c4: + roles: + agg: + args: + max-num-seqs: 8 + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + args: + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp8_c16: + roles: + agg: + args: + max-num-seqs: 32 + benchmark: + env: + CONC: '16' + +override_tp8_c32: + roles: + agg: + args: + max-num-seqs: 64 + benchmark: + env: + CONC: '32' + +override_tp8_c40: + roles: + agg: + args: + max-num-seqs: 80 + benchmark: + env: + CONC: '40' + +override_tp8_c48: + roles: + agg: + args: + max-num-seqs: 96 + benchmark: + env: + CONC: '48' + +override_tp8_c56: + roles: + agg: + args: + max-num-seqs: 112 + benchmark: + env: + CONC: '56' + +override_tp8_c64: + roles: + agg: + args: + max-num-seqs: 128 + benchmark: + env: + CONC: '64' + +override_dep8_c64: + frontend: + type: vllm-router + args: + policy: consistent_hash + request-timeout-secs: 14400 + disable-retries: true + env: + SETUP_PIP_PACKAGES: 'vllm-router==0.1.14' + roles: + agg: + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + prefill-schedule-interval: 8 + long-prefill-token-threshold: 16384 + max-num-seqs: 64 + benchmark: + env: + CONC: '64' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c128: + frontend: + type: vllm-router + args: + policy: consistent_hash + request-timeout-secs: 14400 + disable-retries: true + env: + SETUP_PIP_PACKAGES: 'vllm-router==0.1.14' + roles: + agg: + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + prefill-schedule-interval: 8 + long-prefill-token-threshold: 16384 + max-num-seqs: 128 + benchmark: + env: + CONC: '128' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_c192: + frontend: + type: vllm-router + args: + policy: consistent_hash + request-timeout-secs: 14400 + disable-retries: true + env: + SETUP_PIP_PACKAGES: 'vllm-router==0.1.14' + roles: + agg: + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + prefill-schedule-interval: 8 + long-prefill-token-threshold: 16384 + max-num-seqs: 192 + benchmark: + env: + CONC: '192' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..c87f4758e3 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,263 @@ +# DeepSeek-V4.1-Flash AgentX on B200 with SGLang native DSpark. The KV cache +# is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-b200-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + # Keep active decode requests progressing while long prefixes are queued. + prefill-decode-interval: 16 + # DSpark is the checkpoint's bundled draft; the block size is its only knob. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + # Admission never exceeds the captured decode graph batch. + cuda-graph-max-bs-decode: 64 + reasoning-parser: auto + tool-call-parser: auto + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV41_REASONING_EFFORT: high + # Row-sharded host Engram tables free HBM for the 1M-context working set. + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC, capped at the graph batch. SWA +# prefix tails scale with CONC (TP4 64x, TP2 128x) within [128, 4096]. TP2 +# needs a larger static fraction and a smaller prefill chunk to fit, and +# expandable segments against fragmentation. Saturation points get a longer +# warmup drain. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 128 + max-running-requests: 2 + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 128 + max-running-requests: 4 + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 256 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 512 + max-running-requests: 16 + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 1024 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 2048 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 4096 + max-running-requests: 64 + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + chunked-prefill-size: 4096 + swa-prefix-tails: 4096 + max-running-requests: 64 + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 128 + max-running-requests: 2 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 256 + max-running-requests: 4 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 512 + max-running-requests: 8 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 1024 + max-running-requests: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..61192ac5ea --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,308 @@ +# DeepSeek-V4.1-Flash AgentX on B300 with SGLang native DSpark. The KV cache +# is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-b300-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:35ea4d321b0735051dcce2599fc495853ad1ef30f6a1908f0a75e74204362d14 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + # Overlap checkpoint reads with weight loading from local NVMe. + weight-loader-prefetch-checkpoints: true + chunked-prefill-size: 4096 + # Keep active decode requests progressing while long prefixes are queued. + prefill-decode-interval: 16 + # DSpark is the checkpoint's bundled draft; the block size is its only knob. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + # Admission never exceeds the captured decode graph batch. + cuda-graph-max-bs-decode: 64 + reasoning-parser: auto + tool-call-parser: auto + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV41_REASONING_EFFORT: high + # Row-sharded host Engram tables free HBM for the 1M-context working set. + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC, capped at the graph batch. SWA +# prefix tails are 64x CONC within [128, 4096]. TP2 raises the static fraction +# (0.80 from c16, 0.85 from c32) and interleaves decode more often from c32. +# Saturation points get a longer warmup drain. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 128 + max-running-requests: 2 + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 128 + max-running-requests: 4 + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 256 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 512 + max-running-requests: 16 + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 1024 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 2048 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 4096 + max-running-requests: 64 + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.7 + swa-prefix-tails: 4096 + max-running-requests: 64 + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.7 + swa-prefix-tails: 128 + max-running-requests: 2 + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.7 + swa-prefix-tails: 128 + max-running-requests: 4 + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.7 + swa-prefix-tails: 256 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.7 + swa-prefix-tails: 512 + max-running-requests: 16 + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.8 + swa-prefix-tails: 1024 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.85 + swa-prefix-tails: 2048 + max-running-requests: 64 + prefill-decode-interval: 4 + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.85 + swa-prefix-tails: 4096 + max-running-requests: 64 + prefill-decode-interval: 4 + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.85 + swa-prefix-tails: 4096 + max-running-requests: 64 + prefill-decode-interval: 4 + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..560eb34bee --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml @@ -0,0 +1,345 @@ +# DeepSeek-V4.1-Flash AgentX on GB200 with SGLang native DSpark. The KV cache +# is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-gb200-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17@sha256:5921361fcf358cdde4df1968c941c14157f418613b099ad7f3e5aeed6427ae15 + precision: fp4 + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + # DSpark is the checkpoint's bundled draft; the block size is its only knob. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + reasoning-parser: auto + tool-call-parser: auto + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV41_REASONING_EFFORT: high + # Row-sharded host Engram tables free HBM for the 1M-context working set. + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + # Cold weight loads from the shared HF cache plus graph capture outlast + # srt-slurm's 1800 s default, as they did the legacy one-hour deadline. + health_check: + max_attempts: 1440 + interval_seconds: 5 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. TP4: static fraction 0.70, 4096-token prefill chunks, +# 64-request graph batch. TP2 needs 0.92, 2048-token chunks, a 16-request graph +# batch and expandable segments to fit. Admission is 2x CONC, capped at the graph +# batch. SWA prefix tails are 64x CONC (TP2 128x), capped at 1024. Decode +# interleaving applies to TP2 and c16. Saturation points get a longer warmup drain. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 64 + cuda-graph-max-bs-decode: 64 + max-running-requests: 2 + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 128 + cuda-graph-max-bs-decode: 64 + max-running-requests: 4 + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 256 + cuda-graph-max-bs-decode: 64 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 512 + cuda-graph-max-bs-decode: 64 + max-running-requests: 16 + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 64 + max-running-requests: 32 + prefill-decode-interval: 16 + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 64 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 64 + max-running-requests: 64 + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 64 + max-running-requests: 64 + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 128 + cuda-graph-max-bs-decode: 16 + max-running-requests: 2 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 256 + cuda-graph-max-bs-decode: 16 + max-running-requests: 4 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 512 + cuda-graph-max-bs-decode: 16 + max-running-requests: 8 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 16 + max-running-requests: 16 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 16 + max-running-requests: 16 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 16 + max-running-requests: 16 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 16 + max-running-requests: 16 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + mem-fraction-static: 0.92 + chunked-prefill-size: 2048 + swa-prefix-tails: 1024 + cuda-graph-max-bs-decode: 16 + max-running-requests: 16 + prefill-decode-interval: 16 + env: + PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True' + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..e002f52cf1 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml @@ -0,0 +1,344 @@ +# DeepSeek-V4.1-Flash AgentX on GB300 with SGLang native DSpark. The KV cache +# is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-gb300-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40 + precision: fp4 + resources: + gpu_type: gb300 + gpus_per_node: 4 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + # Keep active decode requests progressing while long prefixes are queued. + prefill-decode-interval: 16 + mem-fraction-static: 0.8 + # DSpark is the checkpoint's bundled draft; the block size is its only knob. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + # Admission never exceeds the captured decode graph batch. + cuda-graph-max-bs-decode: 64 + reasoning-parser: auto + tool-call-parser: auto + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV41_REASONING_EFFORT: high + SGLANG_DSPARK_OPT_MARKOV_W2_BF16: 'True' + # Cold weight loads from the shared HF cache plus graph capture outlast + # srt-slurm's 1800 s default, as they did the legacy one-hour deadline. + health_check: + max_attempts: 1440 + interval_seconds: 5 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC, capped at the graph batch. SWA +# prefix tails are 64x CONC up to 4096 from c2. TP2 keeps row-sharded Engram +# tables in host DRAM (TP4 fits them in HBM) and doubles the prefill chunk from +# c64. Saturation points get a longer warmup drain. +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 4096 + max-running-requests: 2 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 4096 + swa-prefix-tails: 128 + max-running-requests: 4 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 4096 + swa-prefix-tails: 256 + max-running-requests: 8 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 4096 + swa-prefix-tails: 512 + max-running-requests: 16 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 4096 + swa-prefix-tails: 1024 + max-running-requests: 32 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 4096 + swa-prefix-tails: 2048 + max-running-requests: 64 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 8192 + swa-prefix-tails: 4096 + max-running-requests: 64 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + chunked-prefill-size: 8192 + swa-prefix-tails: 4096 + max-running-requests: 64 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + max-running-requests: 2 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + swa-prefix-tails: 128 + max-running-requests: 4 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + swa-prefix-tails: 256 + max-running-requests: 8 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + swa-prefix-tails: 512 + max-running-requests: 16 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + swa-prefix-tails: 1024 + max-running-requests: 32 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + swa-prefix-tails: 2048 + max-running-requests: 64 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + swa-prefix-tails: 4096 + max-running-requests: 64 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + chunked-prefill-size: 4096 + swa-prefix-tails: 4096 + max-running-requests: 64 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..6f4767f0ce --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml @@ -0,0 +1,257 @@ +# DeepSeek-V4.1-Flash AgentX on H100 with SGLang native DSpark at TP8/EP8, as +# a plain TP8 curve and a DP8-attention sweep. H100 is not in the cookbook's +# hardware table: 80 GB cards cannot hold the resident weights plus the +# row-sharded Engram tables and still leave a KV pool, so the tables move to +# per-rank host shards and the prefill chunk is bounded so the sparse-attention +# indexer's [chunk, context] scoring buffer fits next to the weights at 1M +# context. The KV cache is GPU-resident. +# https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 +base: + schema: 2 + name: dsv41flash-fp4-h100-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40 + precision: fp4 + resources: + gpu_type: h100 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + # Cold loads of the checkpoint and host Engram tables from shared NFS outlast + # the default 1800 s health window (c20 was still loading at 1800 s). + health_check: + interval_seconds: 10 + max_attempts: 360 + # Hardware-specific tiling only; checkpoint data, scales and dtypes are unchanged. + setup_script: dsv41flash-h100-block32.sh + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + # Native MXFP4 Marlin supports Hopper with BF16 activations; dense FP8 + # operators and shipped DSpark precision remain unchanged. + moe-runner-backend: marlin + # 0.7 leaves 24 GB for the 4096-token chunk's working set: 0.8/4096 + # OOMed c8 (run 35304509605), and a 2048 chunk failed c4's latency + # coverage at ~1,000 prefill tok/s (run 35307250127). + mem-fraction-static: 0.7 + chunked-prefill-size: 4096 + # DSpark is the checkpoint's bundled draft; the block size is its only knob. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + # Bound long-prefill decode stalls. + prefill-decode-interval: 16 + reasoning-parser: auto + tool-call-parser: auto + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Long-context indexer masks change allocation sizes across requests. At + # C20 the stock allocator OOMed on a 2.39 GiB mask with 5.61 GiB reserved. + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV41_REASONING_EFFORT: high + # Row-sharded anonymous host Engram tables; the KV cache stays on GPU. + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC (at least TP under DP attention) +# with decode graphs clamped to 64-128. The TP SWA tail reserve is 32x CONC from +# c4 (8x below); the DP pool is per rank, and 64 tails/rank keeps the C16 TP +# aggregate 512-tail reserve. +override_tp8_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 8 + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + args: + max-running-requests: 4 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 16 + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 128 + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 256 + benchmark: + env: + CONC: '8' + +override_tp8_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 512 + benchmark: + env: + CONC: '16' + +override_tp8_c20: + roles: + agg: + args: + max-running-requests: 40 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 640 + benchmark: + env: + CONC: '20' + +# DP8 attention: the shipped MoE DSpark worker requires attn_tp=1. SGLang divides +# the engine-wide 4096-token chunk by DP (512 tokens/rank). The DP-aware router +# hashes AIPerf's per-session correlation ID so prefixes stay on one rank. +override_dp8_c4: + frontend: + type: sglang-router + args: + policy: consistent_hashing + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 8 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 64 + benchmark: + env: + CONC: '4' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + +override_dp8_c8: + frontend: + type: sglang-router + args: + policy: consistent_hashing + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 16 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 64 + benchmark: + env: + CONC: '8' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + +override_dp8_c16: + frontend: + type: sglang-router + args: + policy: consistent_hashing + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 32 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 64 + benchmark: + env: + CONC: '16' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + +override_dp8_c20: + frontend: + type: sglang-router + args: + policy: consistent_hashing + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + args: + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 40 + cuda-graph-max-bs-decode: 64 + swa-prefix-tails: 64 + benchmark: + env: + CONC: '20' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..0b55113278 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml @@ -0,0 +1,308 @@ +# DeepSeek-V4.1-Flash AgentX on H200 with SGLang native DSpark, following the +# published vLLM baseline topologies TP4/EP1 and TP8/EP1. The KV cache is +# GPU-resident. https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 +base: + schema: 2 + name: dsv41flash-fp4-h200-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17 + precision: fp4 + resources: + gpu_type: h200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + # Measured H200 launch configurations: the replicated projection on TP4 and + # TP8, plus the qualified TP8 sharded shapes. + setup_script: dsv41flash-h200-block32.sh + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + data-parallel-size: 1 + expert-parallel-size: 1 + attention-backend: dsv4 + # Native Marlin pads the TP-sharded 2304-wide experts (TP4=576, TP8=288) + # that the SM90 CUTLASS method rejects; both consume the MXFP4 weights. + moe-runner-backend: marlin + # Below the cookbook's 0.8, with a bounded prefill chunk: the sparse + # attention indexer and DSpark prefill buffers scale with chunk times the + # 1M context. The default 16384 chunk exhausted HBM on 66k-99k-token + # prompts, and 8192 OOMed c32 (run 35308550355). + mem-fraction-static: 0.70 + chunked-prefill-size: 4096 + # Keep active decode requests progressing while long prefixes are queued. + prefill-decode-interval: 16 + enable-decoder-swa-bounded-replay: true + # The default 4*max-running-requests retained too few SWA prefix tails: + # C16 exhausted its 94,976-slot SWA pool while the full pool had room. + swa-prefix-tails: 1024 + # DSpark is the checkpoint's bundled draft; the block size is its only knob. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + # A verify step above the captured batch runs eagerly and allocates its + # attention workspace, which OOMed the eval at 128 running requests + # (run 35306704553), so admission never exceeds the graph batch. + cuda-graph-max-bs-decode: 64 + reasoning-parser: auto + tool-call-parser: auto + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV41_REASONING_EFFORT: high + # Move the fp8 Engram tables to row-sharded host memory, freeing ~23 GiB + # of HBM per GPU. With them on GPU, c128 died at startup (run 35304458924). + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC, capped at the decode graph batch; +# saturation points carry a larger in-flight working set than the 30-minute +# warmup drain allows. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 2 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 4 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 8 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 16 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 32 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 64 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 64 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 64 + env: + DSV41_BLOCK32_TP: '4' + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 2 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 4 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 8 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp8_c8: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 16 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 32 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp8_c32: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 64 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c64: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 64 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c128: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 64 + env: + DSV41_BLOCK32_TP: '8' + benchmark: + env: + CONC: '128' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..775db80214 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,114 @@ +# DeepSeek-V4.1-Flash AgentX on MI355X with SGLang native DSpark, following the +# cookbook's MI350X TP4/EP4 cell with radix caching enabled for AgentX prefix +# reuse. The KV cache and TP4 Engram tables are GPU-resident. +# https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 +base: + schema: 2 + name: dsv41flash-fp4-mi355x-sglang-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: lmsysorg/sglang:dev-dsv41-mi35x@sha256:c609f74f01b206af9664fb66da8ebef47642a7328a2478eabeebb352e014661a + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 4 + # C32 still exhausted HBM at 0.80 with 4096-token chunks; reserve + # another 10% of physical HBM for native long-prefill scratch. + mem-fraction-static: 0.7 + # Native FP4 prefill scratch scales with query tokens times context. + chunked-prefill-size: 4096 + # The compressed-KV store computes byte offsets in signed int32. Keep + # ratio-1 pages below 2 GiB without reducing the 1M context limit. + max-total-tokens: 3145728 + # C32 fanout reached 63 running requests before a native HIP illegal + # access. Bound admission; additional client requests remain queued. + max-running-requests: 32 + # DSpark is the checkpoint's bundled draft; the block size is its only knob. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + cuda-graph-max-bs-decode: 64 + # Breakable prefill hit HIP illegal access; prefill runs eagerly. + cuda-graph-backend-prefill: disabled + reasoning-parser: auto + tool-call-parser: auto + # Draft passes under long-context load outlast the 1800 s default watchdog. + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # AgentX measures thinking on, the regime of the golden acceptance curve. + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV41_REASONING_EFFORT: high + # The preview image's native kernels, allocator and tuning CSV; TP4 + # Engram stays GPU-resident as in the official MI350X recipe. + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + SGLANG_USE_AITER: '1' + SGLANG_MOE_PADDING: '1' + AITER_FLYDSL_FORCE_REDUCE: '1' + ROCM_QUICK_REDUCE_QUANTIZATION: NONE + # The allocator only garbage-collects below a 1.0 memory fraction; + # reclaim unused long-prefill blocks for HIP/RCCL at 80% of 99%. + PYTORCH_HIP_ALLOC_CONF: garbage_collection_threshold:0.8,per_process_memory_fraction:0.99 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Full-context corpus for this 1M-context recipe. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point; the saturation point gets a longer warmup drain. +override_tp4_c1: + benchmark: + env: + CONC: '1' + +override_tp4_c2: + benchmark: + env: + CONC: '2' + +override_tp4_c4: + benchmark: + env: + CONC: '4' + +override_tp4_c8: + benchmark: + env: + CONC: '8' + +override_tp4_c16: + benchmark: + env: + CONC: '16' + +override_tp4_c32: + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..82730b3561 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,251 @@ +# DeepSeek-V4.1-Flash AgentX on B200 with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). Engram weights use +# UVA DRAM; the KV cache is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-b200-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + engram-config: '{"cpu_offload":true}' + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + disable-uvicorn-access-log: true + env: + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Graph capture starts at 64 tokens and doubles until it +# covers CONC x (1 + 5 drafts), up to 2048. TP2 leaves ~145 GiB of weights per +# rank, so it caps batched tokens at 4096 (the indexer's 1M-wide logits buffer), +# bounds the scheduler at 2x CONC within [16, 256] (FlashInfer autotune fails +# below 16) and stops capturing above 512 tokens. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 128 + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 256 + benchmark: + env: + CONC: '32' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '64' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 1024 + benchmark: + env: + CONC: '128' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '1' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '2' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '4' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 128 + max-num-batched-tokens: 4096 + max-num-seqs: 32 + benchmark: + env: + CONC: '16' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 256 + max-num-batched-tokens: 4096 + max-num-seqs: 64 + benchmark: + env: + CONC: '32' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 128 + benchmark: + env: + CONC: '64' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 256 + benchmark: + env: + CONC: '128' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..34c77fe338 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,256 @@ +# DeepSeek-V4.1-Flash AgentX on B300 with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). Engram weights use +# UVA DRAM; the KV cache is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-b300-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai:deepseekv41-flash-0909 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + engram-config: '{"cpu_offload":true}' + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + max-num-seqs: 256 + disable-uvicorn-access-log: true + env: + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Piecewise graph capture sizes are multiples of the +# six-token DSpark verification block, denser for small batches, and each +# batched-token limit matches the largest captured graph: CONC <= 4 and TP2 +# CONC 128 capture up to 2046 tokens (the latter at 0.97 memory utilization), +# every other point up to 8190. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}' + max-cudagraph-capture-size: 2046 + max-num-batched-tokens: 2048 + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}' + max-cudagraph-capture-size: 2046 + max-num-batched-tokens: 2048 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}' + max-cudagraph-capture-size: 2046 + max-num-batched-tokens: 2048 + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '32' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '64' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '128' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}' + max-cudagraph-capture-size: 2046 + max-num-batched-tokens: 2048 + benchmark: + env: + CONC: '2' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}' + max-cudagraph-capture-size: 2046 + max-num-batched-tokens: 2048 + benchmark: + env: + CONC: '4' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '8' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '16' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '32' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}' + max-cudagraph-capture-size: 8190 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '64' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + compilation-config: '{"mode":"VLLM_COMPILE","cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046]}' + max-cudagraph-capture-size: 2046 + max-num-batched-tokens: 2048 + gpu-memory-utilization: 0.97 + benchmark: + env: + CONC: '128' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..64e0e7cca0 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml @@ -0,0 +1,251 @@ +# DeepSeek-V4.1-Flash AgentX on GB200 with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). Engram weights use +# UVA DRAM; the KV cache is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-gb200-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 + precision: fp4 + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # Cold loads of the shared-cache checkpoint run past half an hour. + health_check: + interval_seconds: 5 + max_attempts: 1440 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + engram-config: '{"cpu_offload":true}' + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + disable-uvicorn-access-log: true + env: + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Graph capture starts at 64 tokens and doubles until it +# covers CONC x (1 + 5 drafts), up to 2048. TP2 leaves ~145 GiB of weights on +# each 256 GiB GPU, so it caps batched tokens at 4096 (the indexer's 1M-wide +# logits buffer), bounds the scheduler at 2x CONC within [16, 256] (FlashInfer +# autotune fails below 16) and stops capturing above 512 tokens. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 128 + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 256 + benchmark: + env: + CONC: '32' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '64' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 1024 + benchmark: + env: + CONC: '128' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '1' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '2' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '4' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 128 + max-num-batched-tokens: 4096 + max-num-seqs: 32 + benchmark: + env: + CONC: '16' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 256 + max-num-batched-tokens: 4096 + max-num-seqs: 64 + benchmark: + env: + CONC: '32' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 128 + benchmark: + env: + CONC: '64' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 256 + benchmark: + env: + CONC: '128' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..1b5e2787e2 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml @@ -0,0 +1,251 @@ +# DeepSeek-V4.1-Flash AgentX on GB300 with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). Engram weights use +# UVA DRAM; the KV cache is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-gb300-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai:nightly-af1c01499b289be555c475669ba50a88e96d846e + precision: fp4 + resources: + gpu_type: gb300 + gpus_per_node: 4 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # Cold loads of the shared-cache checkpoint run past half an hour. + health_check: + interval_seconds: 5 + max_attempts: 1440 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + engram-config: '{"cpu_offload":true}' + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + disable-uvicorn-access-log: true + env: + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Graph capture starts at 64 tokens and doubles until it +# covers CONC x (1 + 5 drafts), up to 2048. TP2 leaves ~175 GiB of weights on +# each 277 GiB GPU, so it caps batched tokens at 4096 (the indexer's 1M-wide +# logits buffer), bounds the scheduler at 2x CONC within [16, 256] (FlashInfer +# autotune fails below 16) and stops capturing above 512 tokens. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 128 + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 256 + benchmark: + env: + CONC: '32' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '64' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 1024 + benchmark: + env: + CONC: '128' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '1' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '2' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '4' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 128 + max-num-batched-tokens: 4096 + max-num-seqs: 32 + benchmark: + env: + CONC: '16' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 256 + max-num-batched-tokens: 4096 + max-num-seqs: 64 + benchmark: + env: + CONC: '32' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 128 + benchmark: + env: + CONC: '64' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 256 + benchmark: + env: + CONC: '128' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..d9a4abce41 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml @@ -0,0 +1,165 @@ +# DeepSeek-V4.1-Flash AgentX on H100 with vLLM native DSpark +# (https://github.com/vllm-project/recipes/blob/main/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml). +# At 1M context the sparse-attention indexer allocates a [max-num-batched-tokens, +# max-model-len] fp8 logits buffer during startup profiling (8192 x 1048576 x 2 B = +# 16 GiB), which OOMs next to ~36 GiB/GPU of weights on 80 GB cards. Capping +# batched tokens shrinks it; capping max-model-len would force the 256k-capped +# corpus onto a 1M-context model. Engram weights use UVA DRAM; the KV cache is +# GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-h100-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 + precision: fp4 + resources: + gpu_type: h100 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + engram-config: '{"cpu_offload":true}' + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + # The indexer buffer scales linearly with batched tokens: 4096 puts it at + # 8 GiB, and anything larger did not fit next to the weights on this SKU. + max-num-batched-tokens: 4096 + gpu-memory-utilization: 0.92 + disable-uvicorn-access-log: true + env: + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + # The indexer buffer is large enough that allocator fragmentation costs a KV block. + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + PYTHONUNBUFFERED: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. The scheduler batch is 2x CONC (AgentX subagent +# fan-out) instead of vLLM's 1024 default; graph capture is the smallest power of +# two covering that batch x (1 + 5 drafts), up to 2048. +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 2 + max-cudagraph-capture-size: 16 + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 4 + max-cudagraph-capture-size: 32 + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 8 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 16 + max-cudagraph-capture-size: 128 + benchmark: + env: + CONC: '8' + +override_tp8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 32 + max-cudagraph-capture-size: 256 + benchmark: + env: + CONC: '16' + +override_tp8_c20: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 40 + max-cudagraph-capture-size: 256 + benchmark: + env: + CONC: '20' + +override_tp8_c24: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 48 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '24' + +override_tp8_c28: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-num-seqs: 56 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '28' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..2818ab0c17 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml @@ -0,0 +1,257 @@ +# DeepSeek-V4.1-Flash AgentX on H200 with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). Engram weights use +# UVA DRAM; the KV cache is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-h200-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 + precision: fp4 + resources: + gpu_type: h200 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 720 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + engram-config: '{"cpu_offload":true}' + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + # The HF cache here is a VIRTIOFS mount, which vLLM does not treat as a + # network FS, so it memory-maps the 475 GiB checkpoint lazily: 19/48 shards + # loaded in the 3600 s readiness window (run 35012494184). Stream the shards + # into page cache with parallel readers first. + safetensors-load-strategy: prefetch + disable-uvicorn-access-log: true + env: + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Graph capture starts at 64 tokens (DSpark5 verification +# for concurrent subagents at c1-c4) and doubles until it covers CONC x (1 + 5 +# drafts), up to 2048. TP4 leaves ~15 GiB of KV per 141 GB GPU, so it caps +# batched tokens at 4096 (the indexer's 1M-wide logits buffer), bounds the +# scheduler at 2x CONC within [16, 256] (FlashInfer autotune fails below 16) and +# stops capturing above 512 tokens. +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 64 + benchmark: + env: + CONC: '8' + +override_tp8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 128 + benchmark: + env: + CONC: '16' + +override_tp8_c32: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 256 + benchmark: + env: + CONC: '32' + +override_tp8_c64: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '64' + +override_tp8_c128: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-cudagraph-capture-size: 1024 + benchmark: + env: + CONC: '128' + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 4096 + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 128 + max-num-batched-tokens: 4096 + max-num-seqs: 32 + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 256 + max-num-batched-tokens: 4096 + max-num-seqs: 64 + benchmark: + env: + CONC: '32' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 128 + benchmark: + env: + CONC: '64' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 4096 + max-num-seqs: 256 + benchmark: + env: + CONC: '128' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..f2f1956224 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml @@ -0,0 +1,214 @@ +# DeepSeek-V4.1-Flash AgentX on MI300X (gfx942) with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). The KV cache is +# GPU-resident; the Engram tables stay on GPU at TP8 and move to pinned host +# memory below TP8, where the checkpoint share already fills a 192 GB card. +base: + schema: 2 + name: dsv41flash-fp4-mi300x-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984 + precision: fp4 + resources: + gpu_type: mi300x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + # The upstream recipe's name resolves to the Triton MXFP4 W4A16 kernel + # on gfx942. + moe-backend: aiter + gpu-memory-utilization: 0.9 + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. Adaptive + # verification trims requests on device, which the ROCm DeepSeek-V4 + # indexer does not support. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + # Upstream picks 1024 on GPUs with >= 160 GiB, and 2x CONC starves + # subagent fan-out at low CONC. Graph capture covers 128 x (1 + 5 drafts). + max-num-seqs: 128 + max-cudagraph-capture-size: 1024 + # The sparse-attention indexer allocates a [batched-tokens, 1M] fp8 + # logits buffer at startup. + max-num-batched-tokens: 8192 + # The ROCm sparse SWA path lacks the replay window clamp; bounded replay + # memory-faulted at the first prefix hit with a replay start. + no-swa-bounded-replay: true + # Piecewise capture segfaulted on gfx942; capture only full decode + # graphs and run prefill eagerly. + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + disable-uvicorn-access-log: true + env: + VLLM_ROCM_USE_AITER: '1' + VLLM_ROCM_USE_AITER_MOE: '1' + AITER_TRITON_LOG_LEVEL: ERROR + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + OMP_NUM_THREADS: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Full-context corpus for this 1M-context recipe. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. There is no TP2 arm: a 192 GB card measured negative KV +# memory at TP2 (run 35671005506). +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '8' + +override_tp8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '16' + +override_tp8_c32: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '32' + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '32' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..ec9b792ede --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml @@ -0,0 +1,299 @@ +# DeepSeek-V4.1-Flash AgentX on MI325X (gfx942) with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). The KV cache is +# GPU-resident; the Engram tables stay on GPU at TP8 and move to pinned host +# memory below TP8, where the checkpoint share already fills a 256 GB card. +base: + schema: 2 + name: dsv41flash-fp4-mi325x-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984 + precision: fp4 + resources: + gpu_type: mi325x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + # The upstream recipe's name resolves to the Triton MXFP4 W4A16 kernel + # on gfx942. + moe-backend: aiter + gpu-memory-utilization: 0.9 + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. Adaptive + # verification trims requests on device, which the ROCm DeepSeek-V4 + # indexer does not support. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + # Upstream picks 1024 on GPUs with >= 160 GiB, and 2x CONC starves + # subagent fan-out at low CONC. Graph capture covers 128 x (1 + 5 drafts). + max-num-seqs: 128 + max-cudagraph-capture-size: 1024 + # The sparse-attention indexer allocates a [batched-tokens, 1M] fp8 + # logits buffer at startup. + max-num-batched-tokens: 16384 + # The ROCm sparse SWA path lacks the replay window clamp; bounded replay + # memory-faulted at the first prefix hit with a replay start. + no-swa-bounded-replay: true + # Piecewise capture segfaulted on gfx942; capture only full decode + # graphs and run prefill eagerly. + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + disable-uvicorn-access-log: true + env: + VLLM_ROCM_USE_AITER: '1' + VLLM_ROCM_USE_AITER_MOE: '1' + AITER_TRITON_LOG_LEVEL: ERROR + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + OMP_NUM_THREADS: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Full-context corpus for this 1M-context recipe. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. TP2 leaves ~145 GiB of offloaded weights per rank, so it +# halves the indexer buffer, bounds the scheduler at 2x CONC within [16, 256] +# and stops capturing above 512 tokens. +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '8' + +override_tp8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '16' + +override_tp8_c32: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + engram-config: '{"cpu_offload":false}' + benchmark: + env: + CONC: '32' + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + benchmark: + env: + CONC: '32' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-num-batched-tokens: 8192 + max-num-seqs: 16 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '1' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-num-batched-tokens: 8192 + max-num-seqs: 16 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '2' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-num-batched-tokens: 8192 + max-num-seqs: 16 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '4' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-num-batched-tokens: 8192 + max-num-seqs: 16 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '8' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-num-batched-tokens: 8192 + max-num-seqs: 32 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '16' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-num-batched-tokens: 8192 + max-num-seqs: 64 + max-cudagraph-capture-size: 512 + benchmark: + env: + CONC: '32' diff --git a/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..9ab84be6dd --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,285 @@ +# DeepSeek-V4.1-Flash AgentX on MI355X with vLLM native DSpark +# (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash). The KV cache is +# GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-mi355x-vllm-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: vllm/vllm-openai-rocm:nightly-rocm100-7f1a5398e9610d96c473931a26c0e12bbe0d0423 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + language-model-only: true + tokenizer-mode: deepseek_v41 + tool-call-parser: deepseek_v41 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v41 + # The plain name lets the CK a8w4 MoE kernel win vLLM's priority list; + # aiter_triton_mxfp4_bf16 would force the W4A16 Triton kernel. + moe-backend: aiter + gpu-memory-utilization: 0.9 + # Five-token DSpark with probabilistic drafting. Throughput runs replace + # block rejection with the golden acceptance length. Adaptive + # verification trims requests on device, which the ROCm DeepSeek-V4 + # indexer does not support. + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' + max-model-len: 1048576 + # The ROCm sparse SWA path lacks the replay window clamp; bounded replay + # memory-faulted at the first prefix hit with a replay start. + no-swa-bounded-replay: true + disable-uvicorn-access-log: true + env: + VLLM_ROCM_USE_AITER: '1' + VLLM_ROCM_USE_AITER_MOE: '1' + AITER_TRITON_LOG_LEVEL: ERROR + # DeepseekV41ForCausalLM is not torch-compiled upstream; breakable + # graphs keep FULL_AND_PIECEWISE capture working. + VLLM_USE_BREAKABLE_CUDAGRAPH: '1' + OMP_NUM_THREADS: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_USE_RUST_FRONTEND: '1' + PYTHONUNBUFFERED: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Full-context corpus for this 1M-context recipe. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. Engram tables move to pinned host memory at TP2 and +# at c128, where resident tables leave too little KV. Graph capture covers +# 2x CONC (at least 128 sequences) x (1 + 5 drafts), up to 2048 tokens. The +# prefill chunk shrinks at TP2 from c64 and at c128 to buy KV room, and where +# it drops below 1024 x 6 slots the scheduler is capped at the captured shape. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":false}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":false}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":false}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":false}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '8' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":false}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '16' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":false}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '32' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":false}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '64' + +override_tp4_c128: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '128' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '1' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '2' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '4' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '8' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '16' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 16384 + benchmark: + env: + CONC: '32' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 1024 + max-num-batched-tokens: 8192 + benchmark: + env: + CONC: '64' + +override_tp2_c128: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + engram-config: '{"cpu_offload":true}' + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 4096 + max-num-seqs: 256 + benchmark: + env: + CONC: '128' diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..e5672aa6f3 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,135 @@ +# GLM-5.2 MXFP4 AgentX on MI355X with ATOM MTP, TP-only with GPU-resident KV. +# The DCP4 LMCache band stays on the legacy script: srtctl reserves ATOM's +# kv-transfer-config for disaggregated workers. +base: + schema: 2 + name: glm5.2-fp4-mi355x-atom-agentic + model: + path: hf:amd/GLM-5.2-MXFP4 + container: rocm/atom-dev:nightly_202609211553 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: atomesh + container_image: rocm/atom-dev@sha256:f00a9dc588a380038fdad54cd57a956043cb316ddda54ebb1b62a24f8caa343b + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: atom + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: amd/GLM-5.2-MXFP4 + gpu-memory-utilization: 0.95 + enable_prefix_caching: true + # model.layers.78 is the MTP head, shipped in BF16; excluding it keeps + # online ptpc_fp8 quantization off the draft while experts stay MXFP4. + online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","model.layers.[0-9].mlp.*expert*","model.layers.[1-6][0-9].mlp.*expert*","model.layers.7[0-7].mlp.*expert*","model.layers.78.*"]}' + max-num-batched-tokens: 16384 + kv_cache_dtype: fp8 + index_cache_dtype: fp4 + block-size: 64 + method: mtp + env: + PYTHONNOUSERSITE: '1' + AITER_QUICK_REDUCE_QUANTIZATION: INT4 + AITER_USE_FLYDSL_MOE_SORTING: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: amd/GLM-5.2-MXFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. Admission is 2x CONC with graphs captured up to it. The +# draft is five tokens through concurrency 8 and four above it. +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 4 + cudagraph-capture-sizes: '[1,2,4]' + num-speculative-tokens: 5 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 8 + cudagraph-capture-sizes: '[1,2,4,8]' + num-speculative-tokens: 5 + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 16 + cudagraph-capture-sizes: '[1,2,4,8,12,16]' + num-speculative-tokens: 5 + benchmark: + env: + CONC: '8' + +override_tp4_c10: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 20 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20]' + num-speculative-tokens: 4 + benchmark: + env: + CONC: '10' + +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + max-num-seqs: 2 + cudagraph-capture-sizes: '[1,2]' + num-speculative-tokens: 5 + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + max-num-seqs: 4 + cudagraph-capture-sizes: '[1,2,4]' + num-speculative-tokens: 5 + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + max-num-seqs: 8 + cudagraph-capture-sizes: '[1,2,4,8]' + num-speculative-tokens: 5 + benchmark: + env: + CONC: '4' diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..8add44971e --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,128 @@ +# GLM-5.2 NVFP4 AgentX on B200 with SGLang EAGLE MTP off the checkpoint's nextn +# head: the TP8 low-latency arm (fp8 KV, cutedsl bf16 GEMM) with HiCache +# host-DRAM offload at every point. +base: + schema: 2 + name: glm5.2-fp4-b200-sglang-agentic + model: + path: hf:nvidia/GLM-5.2-NVFP4 + container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 1 + kv-cache-dtype: fp8_e4m3 + bf16-gemm-backend: cutedsl + max-prefill-tokens: 8192 + quantization: modelopt_fp4 + # GLM-5.2 emits the GLM-4.7 tool-call format; glm45 leaves calls as raw text. + tool-call-parser: glm47 + reasoning-parser: glm45 + chunked-prefill-size: 8192 + # B200 HBM leaves ~31 GB outside the static share for EAGLE verification, + # graph capture and DSA indexer temporaries. + mem-fraction-static: 0.83 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + watchdog-timeout: 1800 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + # Keep FlashInfer/Triton/Inductor/JIT caches off the shared home. + SGLANG_CACHE_DIR: /tmp/sglang-cache + # Outlast AIPerf's pooled keep-alive connections. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/GLM-5.2-NVFP4 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +# One variant per point. Admission is 2x CONC and the graph batch matches it, +# capped at 64. MLA ranks hold complete per-token KV, so ratio 0.75 gives too +# few host slots past C8; C12/C16 pin a 169 GB/rank pool (~1,662 GB across TP8 +# with the DSA indexer, within the 1,731 GB budget). +override_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 2 + hicache-ratio: 0.75 + benchmark: + env: + CONC: '1' + +override_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 8 + hicache-ratio: 0.75 + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 16 + hicache-ratio: 0.75 + benchmark: + env: + CONC: '8' + +override_c12: + roles: + agg: + args: + max-running-requests: 24 + cuda-graph-max-bs: 24 + hicache-size: 169 + benchmark: + env: + CONC: '12' + +override_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 32 + hicache-size: 169 + benchmark: + env: + CONC: '16' diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..5aa940cc33 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml @@ -0,0 +1,128 @@ +# GLM-5.2 FP8 AgentX on B200 with SGLang EAGLE MTP off the checkpoint's nextn +# head: the TP8 low-latency arm (fp8 KV, cutedsl bf16 GEMM) with HiCache +# host-DRAM offload at every point. +base: + schema: 2 + name: glm5.2-fp8-b200-sglang-agentic + model: + path: hf:zai-org/GLM-5.2-FP8 + container: lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b + precision: fp8 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 1 + kv-cache-dtype: fp8_e4m3 + bf16-gemm-backend: cutedsl + max-prefill-tokens: 8192 + quantization: fp8 + # GLM-5.2 emits the GLM-4.7 tool-call format; glm45 leaves calls as raw text. + tool-call-parser: glm47 + reasoning-parser: glm45 + chunked-prefill-size: 8192 + # B200 HBM leaves ~31 GB outside the static share for EAGLE verification, + # graph capture and DSA indexer temporaries. + mem-fraction-static: 0.83 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + watchdog-timeout: 1800 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + # Keep FlashInfer/Triton/Inductor/JIT caches off the shared home. + SGLANG_CACHE_DIR: /tmp/sglang-cache + # Outlast AIPerf's pooled keep-alive connections. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: zai-org/GLM-5.2-FP8 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +# One variant per point. Admission is 2x CONC and the graph batch matches it, +# capped at 64. MLA ranks hold complete per-token KV, so ratio 0.75 gives too +# few host slots past C8; C12/C16 pin a 169 GB/rank pool (~1,662 GB across TP8 +# with the DSA indexer, within the 1,731 GB budget). +override_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 2 + hicache-ratio: 0.75 + benchmark: + env: + CONC: '1' + +override_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 8 + hicache-ratio: 0.75 + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 16 + hicache-ratio: 0.75 + benchmark: + env: + CONC: '8' + +override_c12: + roles: + agg: + args: + max-running-requests: 24 + cuda-graph-max-bs: 24 + hicache-size: 169 + benchmark: + env: + CONC: '12' + +override_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 32 + hicache-size: 169 + benchmark: + env: + CONC: '16' diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..e32b72696a --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,181 @@ +# GLM-5.2 NVFP4 AgentX on B300 with SGLang EAGLE off the checkpoint's nextn +# head: the low-latency TP8 arm (fp8 KV, cutedsl bf16 GEMM). HiCache spills +# evicted prefixes to host DRAM; without it the working set outgrows HBM past +# c8 and the radix hit rate collapses. +base: + schema: 2 + name: glm5.2-fp4-b300-sglang-agentic + model: + path: hf:nvidia/GLM-5.2-NVFP4 + container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 1 + kv-cache-dtype: fp8_e4m3 + bf16-gemm-backend: cutedsl + max-prefill-tokens: 8192 + quantization: modelopt_fp4 + # GLM-5.2 emits the GLM-4.7 tool-call format. + tool-call-parser: glm47 + reasoning-parser: glm45 + chunked-prefill-size: 8192 + mem-fraction-static: 0.85 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + # MLA-family: every TP rank holds complete per-token KV, so the host + # pool is an absolute 270 GB/rank; the DSA indexer pool rides on top. + enable-hierarchical-cache: true + hicache-size: 270 + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + watchdog-timeout: 1800 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/GLM-5.2-NVFP4 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2399' + +# One variant per point: admission is 2x CONC and the graph batch matches it, +# capped at 64. +override_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 2 + benchmark: + env: + CONC: '1' + +override_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '8' + +override_c12: + roles: + agg: + args: + max-running-requests: 24 + cuda-graph-max-bs: 24 + benchmark: + env: + CONC: '12' + +override_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '16' + +override_c20: + roles: + agg: + args: + max-running-requests: 40 + cuda-graph-max-bs: 40 + benchmark: + env: + CONC: '20' + +override_c24: + roles: + agg: + args: + max-running-requests: 48 + cuda-graph-max-bs: 48 + benchmark: + env: + CONC: '24' + +override_c28: + roles: + agg: + args: + max-running-requests: 56 + cuda-graph-max-bs: 56 + benchmark: + env: + CONC: '28' + +override_c32: + roles: + agg: + args: + max-running-requests: 64 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '32' + +override_c40: + roles: + agg: + args: + max-running-requests: 80 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '40' + +override_c48: + roles: + agg: + args: + max-running-requests: 96 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '48' diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..cfcbf7d3f1 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml @@ -0,0 +1,123 @@ +# GLM-5.2 FP8 AgentX on B300 with SGLang EAGLE off the checkpoint's nextn +# head: the low-latency TP8 arm (fp8 KV, cutedsl bf16 GEMM). HiCache spills +# evicted prefixes to host DRAM; without it the working set outgrows HBM past +# c8 and the radix hit rate collapses. +base: + schema: 2 + name: glm5.2-fp8-b300-sglang-agentic + model: + path: hf:zai-org/GLM-5.2-FP8 + container: lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b + precision: fp8 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 1 + kv-cache-dtype: fp8_e4m3 + bf16-gemm-backend: cutedsl + max-prefill-tokens: 8192 + quantization: fp8 + # GLM-5.2 emits the GLM-4.7 tool-call format. + tool-call-parser: glm47 + reasoning-parser: glm45 + chunked-prefill-size: 8192 + mem-fraction-static: 0.85 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + # MLA-family: every TP rank holds complete per-token KV, so the host + # pool is an absolute 270 GB/rank; the DSA indexer pool rides on top. + enable-hierarchical-cache: true + hicache-size: 270 + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + watchdog-timeout: 1800 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # Keep FlashInfer, Triton and JIT caches allocation-local. + SGLANG_CACHE_DIR: /tmp/sglang-cache + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: zai-org/GLM-5.2-FP8 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2399' + +# One variant per point: admission is 2x CONC and the graph batch matches it, +# capped at 64. +override_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 2 + benchmark: + env: + CONC: '1' + +override_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '8' + +override_c12: + roles: + agg: + args: + max-running-requests: 24 + cuda-graph-max-bs: 24 + benchmark: + env: + CONC: '12' + +override_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '16' diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..f329216a2d --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml @@ -0,0 +1,142 @@ +# GLM-5.2 FP8 AgentX on MI325X with SGLang native EAGLE MTP. The KV cache is +# GPU-resident. +base: + schema: 2 + name: glm5.2-fp8-mi325x-sglang-agentic + model: + path: hf:zai-org/GLM-5.2-FP8 + container: lmsysorg/sglang:v0.5.19-rocm720-mi30x + precision: fp8 + resources: + gpu_type: mi325x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + # Loading plus first-request kernel tuning outlasts the 1800 s default: + # MI300X Qwen3.5 was ready after ~27 minutes (run 36160403972). + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 1 + dsa-prefill-backend: tilelang + dsa-decode-backend: tilelang + dsa-topk-backend: torch + kv-cache-dtype: bfloat16 + tool-call-parser: glm47 + reasoning-parser: glm45 + context-length: 1048576 + max-total-tokens: 1048576 + chunked-prefill-size: 131072 + mem-fraction-static: 0.85 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + env: + PYTHONNOUSERSITE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # The MI30X image's DSA top-k v2 JIT includes CUDA-only headers when + # compiled for gfx942. Use the portable Torch path, no fused top-k. + SGLANG_DSA_FUSE_TOPK: 'false' + SGLANG_OPT_USE_TOPK_V2: 'false' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: zai-org/GLM-5.2-FP8 + # GLM-5.2 natively supports 1M context, so replay the complete corpus. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + # Long responses admitted near the end of the measurement window can take + # minutes to finish; bound their drain without extending admission. + AIPERF_BENCHMARK_GRACE_PERIOD: '1800' + +# One variant per point. Admission and the decode graph batch are 2x CONC. +override_tp8_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 2 + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + args: + max-running-requests: 4 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '2' + +override_tp8_c3: + roles: + agg: + args: + max-running-requests: 6 + cuda-graph-max-bs: 6 + benchmark: + env: + CONC: '3' + +override_tp8_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '4' + +override_tp8_c5: + roles: + agg: + args: + max-running-requests: 10 + cuda-graph-max-bs: 10 + benchmark: + env: + CONC: '5' + +override_tp8_c6: + roles: + agg: + args: + max-running-requests: 12 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '6' + +override_tp8_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '8' diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..d2469415cc --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,203 @@ +# GLM-5.2 FP4 AgentX on MI355X with SGLang EAGLE MTP. +base: + schema: 2 + name: glm5.2-fp4-mi355x-sglang-agentic + model: + path: hf:amd/GLM-5.2-MXFP4 + container: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260923 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: amd/GLM-5.2-MXFP4 + trust-remote-code: true + kv-cache-dtype: fp8_e4m3 + dsa-prefill-backend: triton + dsa-decode-backend: triton + # GLM-5.2 emits GLM-4.7-style tool calls; glm45 keeps hybrid thinking + # in reasoning_content. + tool-call-parser: glm47 + reasoning-parser: glm45 + # 32k chunks interleave decode between prefill chunks and keep per-chunk + # activation headroom low enough for 0.85. + chunked-prefill-size: 32768 + mem-fraction-static: 0.85 + speculative-algorithm: EAGLE + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + watchdog-timeout: 1800 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # The DSA indexer uses the precompiled HIP Top-K op. + SGLANG_OPT_USE_TOPK_V2: 'false' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: amd/GLM-5.2-MXFP4 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC; the decode graph batch follows it +# up to 64. The TP4 HiCache tier is 180 GB per rank. +override_tp4_c4_hicache: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_through + benchmark: + env: + CONC: '4' + KV_OFFLOADING: dram + +override_tp4_c8_hicache: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_through + benchmark: + env: + CONC: '8' + KV_OFFLOADING: dram + +override_tp4_c10_hicache: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + max-running-requests: 20 + cuda-graph-max-bs-decode: 20 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_through + benchmark: + env: + CONC: '10' + KV_OFFLOADING: dram + +override_tp4_c12_hicache: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + max-running-requests: 24 + cuda-graph-max-bs-decode: 24 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_through + benchmark: + env: + CONC: '12' + KV_OFFLOADING: dram + +override_tp4_c14_hicache: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + max-running-requests: 28 + cuda-graph-max-bs-decode: 28 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_through + benchmark: + env: + CONC: '14' + KV_OFFLOADING: dram + +override_tp4_c16_hicache: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_through + benchmark: + env: + CONC: '16' + KV_OFFLOADING: dram + +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 4 + cuda-graph-max-bs-decode: 4 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none diff --git a/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..28a4b3cc63 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml @@ -0,0 +1,173 @@ +# GLM-5.2 FP8 AgentX on MI355X with SGLang EAGLE MTP. The ~756 GB +# block-FP8 checkpoint is TP8-only; quantization is detected from the checkpoint. +base: + schema: 2 + name: glm5.2-fp8-mi355x-sglang-agentic + model: + path: hf:zai-org/GLM-5.2-FP8 + container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260917 + precision: fp8 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + kv-cache-dtype: fp8_e4m3 + dsa-prefill-backend: tilelang + dsa-decode-backend: tilelang + # GLM-5.2 emits GLM-4.7-style tool calls; glm45 keeps hybrid thinking + # in reasoning_content. + tool-call-parser: glm47 + reasoning-parser: glm45 + # 32k chunks interleave decode between prefill chunks and keep per-chunk + # activation headroom low enough for 0.85. + chunked-prefill-size: 32768 + mem-fraction-static: 0.85 + speculative-algorithm: EAGLE + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + watchdog-timeout: 1800 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # The DSA indexer uses the precompiled HIP Top-K op. + SGLANG_OPT_USE_TOPK_V2: 'false' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: zai-org/GLM-5.2-FP8 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC; the decode graph batch follows it +# up to 64. HiCache holds 1.5x the device KV pool and skips non-reusable blocks. +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c2: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 4 + cuda-graph-max-bs-decode: 4 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c10: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 20 + cuda-graph-max-bs-decode: 20 + benchmark: + env: + CONC: '10' + KV_OFFLOADING: none + +override_tp8_c8_hicache: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through_selective + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + benchmark: + env: + CONC: '8' + KV_OFFLOADING: dram + +override_tp8_c12_hicache: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 24 + cuda-graph-max-bs-decode: 24 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through_selective + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + benchmark: + env: + CONC: '12' + KV_OFFLOADING: dram + +override_tp8_c16_hicache: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through_selective + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + benchmark: + env: + CONC: '16' + KV_OFFLOADING: dram diff --git a/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..ac5021e444 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,104 @@ +# Kimi-K3 MXFP4 AgentX on MI355X with ATOM DSpark: the interactive band, TP8 +# with GPU-resident KV and the deepest published draft (seven tokens). The +# 1.56 TB checkpoint only fits at TP8. The DCP8 LMCache bands stay on the legacy +# script: srtctl reserves ATOM's kv-transfer-config for disaggregated workers. +base: + schema: 2 + name: kimik3-fp4-mi355x-atom-agentic + model: + path: hf:moonshotai/Kimi-K3 + container: rocm/atom-dev:ubuntu24.04_py3.12_pytorch_release_2.10.0_kimi_k3_agentic_0911 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: atomesh + container_image: rocm/atom-dev@sha256:f00a9dc588a380038fdad54cd57a956043cb316ddda54ebb1b62a24f8caa343b + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: atom + setup_script: kimik3-atom-dspark-draft.sh + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: moonshotai/Kimi-K3 + trust-remote-code: true + # Fails closed if the chat template stops rendering a tools payload. + tool-call-parser: kimi_k3 + decode-context-parallel-size: 1 + kv_cache_dtype: fp8 + block-size: 128 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + gpu-memory-utilization: 0.9 + enable_prefix_caching: true + # -1 keeps the prompt-end checkpoint anchor without the fixed-interval grid. + state-checkpoint-interval-tokens: -1 + level: 3 + cudagraph-mode: FULL + # Global ptpc_fp8 online quantization; the trailing entries keep the BF16 + # draft (unprefixed layers.* and context_proj) unquantized. + online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*self_attn.[qkv]_conv1d*","*block_sparse_moe.experts*","*block_sparse_moe.routed_expert_*","*vision_tower*","*mm_projector*","layers.*.self_attn.fused_qkv_a_proj","layers.*.self_attn.q_b_proj","layers.*.self_attn.kv_b_proj","layers.*.self_attn.o_proj","layers.*.mlp.gate_up_proj","layers.*.mlp.down_proj","context_proj"]}' + method: dspark + draft-model: Inferact/Kimi-K3-DSpark + num-speculative-tokens: 7 + env: + # The image bakes in a build-host proxy that nothing serves on a node. + HTTP_PROXY: '' + HTTPS_PROXY: '' + http_proxy: '' + https_proxy: '' + NO_PROXY: localhost,127.0.0.1,::1 + no_proxy: localhost,127.0.0.1,::1 + PYTHONNOUSERSITE: '1' + ATOM_ENABLE_REPLAYSSM: '0' + AITER_REUSE_IDENTICAL_COMM_GROUPS: '0' + AITER_LOG_LEVEL: WARNING + AITER_SITUV2_A4W4: '1' + AITER_QUICK_REDUCE_QUANTIZATION: INT4 + AITER_FLYDSL_STAGE2_FP8: '1' + # Anchor-only state checkpointing. + ATOM_STATE_CHECKPOINT_DEMAND: '0' + ATOM_GDN_SSM_DTYPE: fp16 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: moonshotai/Kimi-K3 + HTTP_PROXY: '' + HTTPS_PROXY: '' + http_proxy: '' + https_proxy: '' + NO_PROXY: localhost,127.0.0.1,::1 + no_proxy: localhost,127.0.0.1,::1 + # Long agentic turns against a 1M context are prefill-bound on the server. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. Full graphs cover every verify batch up to 2x CONC +# times the eight-token verify window. +override_tp8_c1: + roles: + agg: + args: + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' + benchmark: + env: + CONC: '1' + +override_tp8_c4: + roles: + agg: + args: + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]' + benchmark: + env: + CONC: '4' diff --git a/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..e8d5e7039b --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,246 @@ +# Kimi-K3 MXFP4 AgentX on B300 with vLLM: TP8 x DCP8 with TokenspeedMLA and +# Mooncake as the external KV tier. DSpark drafts seven tokens up to CONC 8, +# three at CONC 16 and none above. +base: + schema: 2 + name: kimik3-fp4-b300-vllm-agentic + model: + path: hf:moonshotai/Kimi-K3 + container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # ~1.5 TB of MXFP4 shards may take the full VLLM_ENGINE_READY_TIMEOUT_S. + health_check: + interval_seconds: 10 + max_attempts: 360 + # Embedded Mooncake: each TP rank contributes TOTAL_CPU_DRAM_GB / 8 GB. The + # setup script pins the client to the master's version and fills in the + # node's active RDMA rail. DSXE rails are InfiniBand without a netdev, so the + # transfer engine picks its own GID (a RoCE v2 index 3 does not exist). + setup_script: kimik3-b300-mooncake.sh + services: + - name: mooncake-master + type: mooncake-master + preamble: >- + python3 -m pip install --break-system-packages --quiet --no-cache-dir --no-deps + --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 + args: ["--eviction_high_watermark_ratio=0.95", "--eviction_ratio=0.10"] + options: + store_config: + mode: embedded + metadata_server: P2PHANDSHAKE + global_segment_size: 281GB + local_buffer_size: 4GB + protocol: rdma + device_name: "" + enable_offload: false + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: moonshotai/Kimi-K3 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + load-format: fastsafetensors + moe-backend: auto + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: fp8 + stream-interval: 10 + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + disable-uvicorn-access-log: true + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + env: + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + # These default to auto; name them so the measured DCP a2a path runs. + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + PYTHONNOUSERSITE: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + # Identical prefixes must hash to identical block keys run-to-run. + PYTHONHASHSEED: '42' + # Outlast AIPerf's pooled-connection idle gaps. + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900' + MC_STORE_MEMCPY: '1' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + MC_SLICE_SIZE: '1048576' + MC_WORKERS_PER_CTX: '4' + WITH_NVIDIA_PEERMEM: '0' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: moonshotai/Kimi-K3 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2249' + +# One variant per point. Admission is 2x CONC; CONC 56 and 70 keep more memory +# headroom. Graphs capture (1 + drafts) x 1..min(2x CONC, 128) tokens, then the +# larger powers of two to 8192. Throughput runs switch DSpark to synthetic +# rejection at the golden acceptance length; points above CONC 16 do not draft +# and keep the matrix's mtp label. +override_c1: + roles: + agg: + args: + max-num-seqs: 2 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,64,128,256,512,1024,2048,4096,8192]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '1' + +override_c2: + roles: + agg: + args: + max-num-seqs: 4 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,64,128,256,512,1024,2048,4096,8192]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '2' + +override_c4: + roles: + agg: + args: + max-num-seqs: 8 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,128,256,512,1024,2048,4096,8192]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-num-seqs: 16 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,256,512,1024,2048,4096,8192]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '8' + +override_c16: + roles: + agg: + args: + max-num-seqs: 32 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,256,512,1024,2048,4096,8192]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '16' + +override_c24: + roles: + agg: + args: + max-num-seqs: 48 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,64,128,256,512,1024,2048,4096,8192]}' + benchmark: + env: + CONC: '24' + SPEC_DECODING: mtp + +override_c32: + roles: + agg: + args: + max-num-seqs: 64 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,128,256,512,1024,2048,4096,8192]}' + benchmark: + env: + CONC: '32' + SPEC_DECODING: mtp + +override_c40: + roles: + agg: + args: + max-num-seqs: 80 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,128,256,512,1024,2048,4096,8192]}' + benchmark: + env: + CONC: '40' + SPEC_DECODING: mtp + +override_c48: + roles: + agg: + args: + max-num-seqs: 96 + gpu-memory-utilization: 0.92 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,128,256,512,1024,2048,4096,8192]}' + benchmark: + env: + CONC: '48' + SPEC_DECODING: mtp + +override_c56: + roles: + agg: + args: + max-num-seqs: 112 + gpu-memory-utilization: 0.9 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,128,256,512,1024,2048,4096,8192]}' + benchmark: + env: + CONC: '56' + SPEC_DECODING: mtp + +override_c70: + roles: + agg: + args: + max-num-seqs: 140 + gpu-memory-utilization: 0.9 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,256,512,1024,2048,4096,8192]}' + benchmark: + env: + CONC: '70' + SPEC_DECODING: mtp diff --git a/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..0621caa8e2 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,185 @@ +# Kimi-K3 MXFP4 AgentX on MI355X with vLLM DSpark +# (https://recipes.vllm.ai/moonshotai/Kimi-K3). TP8 only: the 1.56 TB checkpoint +# is ~195 GB per GPU. The KV cache is GPU-resident through c4 and backed by +# vLLM's SimpleCPUOffloadConnector from c8. The DCP8 arm (c44-c70) runs without +# a draft model and stays on the legacy script. +base: + schema: 2 + name: kimik3-fp4-mi355x-vllm-agentic + model: + path: hf:moonshotai/Kimi-K3 + container: vllm/vllm-openai-rocm:nightly-rocm100-af1c01499b289be555c475669ba50a88e96d846e + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # Weights load for up to VLLM_ENGINE_READY_TIMEOUT_S. + health_check: + interval_seconds: 10 + max_attempts: 720 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: moonshotai/Kimi-K3 + trust-remote-code: true + moe-backend: auto + tensor-parallel-size: 8 + load-format: fastsafetensors + gpu-memory-utilization: 0.9 + language-model-only: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + max-model-len: 1048576 + enable-prefix-caching: true + kv-cache-dtype: fp8 + attention-config: '{"mla_prefill_backend":"ROCM_AITER_FA"}' + env: + # Upstream AMD recipe environment. + VLLM_ROCM_AITER_MLA_ASM_PADDING: asm + VLLM_ROCM_USE_AITER: '1' + SAFETENSORS_FAST_GPU: '1' + VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4: '1' + AITER_SITUV2_A8W4: '1' + AITER_BF16_FP8_MOE_BOUND: '0' + AITER_QUICK_REDUCE_QUANTIZATION: INT4 + # The MI355X nodes report MEC firmware 38, below the 177 that fixes the + # RCCL memory reclaim issue. + HSA_NO_SCRATCH_RECLAIM: '1' + # 2.8 TB of weights off a shared mount takes far longer than the default. + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONNOUSERSITE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1200' + VLLM_USE_DIRECT_DCP_A2A: '0' + VLLM_USE_DIRECT_DCP_Q_GATHER: '0' + VLLM_USE_DIRECT_DCP_KV_GATHER: '0' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: moonshotai/Kimi-K3 + # Long agentic turns against a 1M context are prefill-bound on the server. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. The DSpark draft length descends with concurrency +# (7, 5, 4, 4, 3, 3); throughput runs replace block rejection with the golden +# acceptance length for that draft length. Admission is 2x CONC and graph +# capture covers every size up to 2x CONC x (1 + drafts). Breakable +# FULL_AND_PIECEWISE graphs cost KV pool, so only c1 and c4 use them. The +# offload points split the 1799 GB DRAM budget evenly across the eight ranks; +# identical prefixes must hash to identical block keys on every rank. +override_tp8_c1: + roles: + agg: + args: + max-num-seqs: 2 + max-num-batched-tokens: 16384 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":16,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '1' + benchmark: + env: + CONC: '1' + KV_OFFLOADING: 'none' + +override_tp8_c4: + roles: + agg: + args: + max-num-seqs: 8 + max-num-batched-tokens: 8192 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":48,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":5,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '1' + benchmark: + env: + CONC: '4' + KV_OFFLOADING: 'none' + +override_tp8_c8_simple: + roles: + agg: + args: + max-num-seqs: 16 + max-num-batched-tokens: 8192 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":80,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '8' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + +override_tp8_c10_simple: + roles: + agg: + args: + max-num-seqs: 20 + max-num-batched-tokens: 8192 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":100,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '10' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + +override_tp8_c12_simple: + roles: + agg: + args: + max-num-seqs: 24 + max-num-batched-tokens: 8192 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":96,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '12' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' + +override_tp8_c14_simple: + roles: + agg: + args: + max-num-seqs: 28 + max-num-batched-tokens: 8192 + compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":112,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112]}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":3,"method":"dspark","attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}' + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + PYTHONHASHSEED: '42' + benchmark: + env: + CONC: '14' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1799' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..df6b8d5165 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,208 @@ +# MiniMax-M3 MXFP4 AgentX on MI355X with ATOM EAGLE3 (GQA draft, three tokens) +# and GPU-resident KV at TP2 and TP4. The LMCache bands stay on the legacy +# script: srtctl reserves ATOM's kv-transfer-config for disaggregated workers. +base: + schema: 2 + name: minimaxm3-fp4-mi355x-atom-agentic + model: + path: hf:amd/MiniMax-M3-MXFP4 + container: rocm/atom-dev:nightly_202609171455 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: atomesh + container_image: rocm/atom-dev@sha256:f00a9dc588a380038fdad54cd57a956043cb316ddda54ebb1b62a24f8caa343b + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: atom + setup_script: minimaxm3-atom-eagle3-deps.sh + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: amd/MiniMax-M3-MXFP4 + trust-remote-code: true + kv_cache_dtype: fp8 + block-size: 128 + max-num-batched-tokens: 32768 + gpu-memory-utilization: 0.95 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,22,24,26,28,30,32,34,36,40,48,56,64]' + index-cache-dtype: fp8 + online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}' + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + method: eagle3 + draft-model: Inferact/MiniMax-M3-EAGLE3-GQA + num-speculative-tokens: 3 + env: + PYTHONNOUSERSITE: '1' + AITER_LOG_LEVEL: WARNING + AITER_SITUV2_A4W4: '1' + AITER_QUICK_REDUCE_QUANTIZATION: INT4 + AITER_FLYDSL_STAGE2_FP8: '1' + ATOM_FORCE_ATTN_TRITON: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: amd/MiniMax-M3-MXFP4 + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. Admission is 2x CONC. TP4 shards the indexer across +# ranks at the measured points (15, 20, 24, 28, 32). +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 2 + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 4 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 8 + benchmark: + env: + CONC: '4' + +override_tp4_c5: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 10 + benchmark: + env: + CONC: '5' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp4_c10: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 20 + benchmark: + env: + CONC: '10' + +override_tp4_c12: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 24 + benchmark: + env: + CONC: '12' + +override_tp4_c15: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 30 + env: + ATOM_M3_INDEXER_CP: '1' + benchmark: + env: + CONC: '15' + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 40 + env: + ATOM_M3_INDEXER_CP: '1' + benchmark: + env: + CONC: '20' + +override_tp4_c24: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 48 + env: + ATOM_M3_INDEXER_CP: '1' + benchmark: + env: + CONC: '24' + +override_tp4_c28: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 56 + env: + ATOM_M3_INDEXER_CP: '1' + benchmark: + env: + CONC: '28' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 64 + env: + ATOM_M3_INDEXER_CP: '1' + benchmark: + env: + CONC: '32' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + max-num-seqs: 2 + benchmark: + env: + CONC: '1' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + max-num-seqs: 4 + benchmark: + env: + CONC: '2' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..987e19242c --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,436 @@ +# MiniMax-M3 NVFP4 AgentX on B200 with TRT-LLM EAGLE3-GQA. DRAM KV offload +# uses TRT-LLM's native secondary pool, pinned per topology rather than derived +# from TOTAL_CPU_DRAM_GB. +base: + schema: 2 + name: minimaxm3-fp4-b200-trtllm-agentic + model: + path: hf:nvidia/MiniMax-M3-NVFP4 + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: trtllm_serve + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: trtllm + served_model_name: nvidia/MiniMax-M3-NVFP4 + # Keep Prometheus without rc23's per-step timing collector and accept BFCL's + # store=false field. + setup_script: minimaxm3-trtllm-rc23.sh + # TP8 loads, autotunes and captures graphs past the 30-minute default. + health_check: + max_attempts: 1440 + interval_seconds: 5 + roles: + agg: + nodes: 1 + workers: 1 + # Server-layer flag; the local checkpoint mounts at /model. + extra_args: [--chat_template, /model/chat_template.jinja] + args: + moe_expert_parallel_size: 1 + max_seq_len: 1048576 + max_num_tokens: 16384 + cuda_graph_config: + enable_padding: true + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: [1, 512, 1024, 2048] + enable_userbuffers: true + max_num_streams: 3 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + free_gpu_memory_fraction: 0.94 + enable_block_reuse: true + block_reuse_policy: per_conversation + tokens_per_block: 128 + use_kv_cache_manager_v2: true + dtype: fp8 + event_buffer_max_size: 0 + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + enable_chunked_prefill: true + enable_autotuner: true + trust_remote_code: true + reasoning_parser: minimax_m3 + stream_interval: 20 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: true + num_postprocess_workers: 8 + enable_attention_dp: false + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TLLM_PROFILE_LOG_RANKS: all + PYTHONNOUSERSITE: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TQDM_DISABLE: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/MiniMax-M3-NVFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + KV_OFFLOADING: dram + +# One variant per point. The batch cap is CONC; graphs capture every batch up +# to 20 and a thinned ladder above. The host pool is 250 GiB at TP4 and +# 200 GiB at TP8. +override_tp4_c5: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 5 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '5' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c10: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 10 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '10' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c15: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 15 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '15' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 20 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '20' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c25: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 25 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 17 + - 19 + - 21 + - 23 + - 25 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '25' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c30: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 30 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 27 + - 30 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '30' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c35: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 35 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 14 + - 17 + - 20 + - 23 + - 26 + - 29 + - 32 + - 35 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '35' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c40: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 40 + cuda_graph_config: + batch_sizes: + - 2 + - 4 + - 6 + - 8 + - 10 + - 12 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 26 + - 28 + - 30 + - 32 + - 34 + - 36 + - 38 + - 40 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '40' + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_c45: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 45 + cuda_graph_config: + batch_sizes: + - 2 + - 4 + - 6 + - 8 + - 10 + - 12 + - 14 + - 16 + - 18 + - 21 + - 24 + - 27 + - 30 + - 33 + - 36 + - 39 + - 42 + - 45 + kv_cache_config: + host_cache_size: 268435456000 + benchmark: + env: + CONC: '45' + TOTAL_CPU_DRAM_GB: '865' + +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor_parallel_size: 8 + max_batch_size: 1 + cuda_graph_config: + batch_sizes: + - 1 + kv_cache_config: + host_cache_size: 214748364800 + benchmark: + env: + CONC: '1' + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_c5: + roles: + agg: + gpus: 8 + args: + tensor_parallel_size: 8 + max_batch_size: 5 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + kv_cache_config: + host_cache_size: 214748364800 + benchmark: + env: + CONC: '5' + TOTAL_CPU_DRAM_GB: '1731' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..8f2a74bbdd --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,438 @@ +# MiniMax-M3 NVFP4 AgentX on B300 with TensorRT-LLM EAGLE3-GQA. DRAM KV offload +# uses TRT-LLM's native secondary-memory pool, pinned per topology. +base: + schema: 2 + name: minimaxm3-fp4-b300-trtllm-agentic + model: + path: hf:nvidia/MiniMax-M3-NVFP4 + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: trtllm_serve + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: trtllm + served_model_name: nvidia/MiniMax-M3-NVFP4 + setup_script: minimaxm3-trtllm-agentx.sh + roles: + agg: + nodes: 1 + workers: 1 + extra_args: [--chat_template, /model/chat_template.jinja] + args: + moe_expert_parallel_size: 1 + max_seq_len: 1048576 + max_num_tokens: 16384 + cuda_graph_config: + enable_padding: true + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: [1, 512, 1024, 2048] + enable_userbuffers: true + max_num_streams: 3 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + free_gpu_memory_fraction: 0.94 + enable_block_reuse: true + block_reuse_policy: per_conversation + tokens_per_block: 128 + use_kv_cache_manager_v2: true + dtype: fp8 + event_buffer_max_size: 0 + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + enable_chunked_prefill: true + enable_autotuner: true + trust_remote_code: true + reasoning_parser: minimax_m3 + stream_interval: 20 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: true + num_postprocess_workers: 8 + enable_attention_dp: false + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TLLM_PROFILE_LOG_RANKS: all + PYTHONNOUSERSITE: '1' + TRTLLM_ENABLE_PDL: '1' + ENROOT_ALLOW_DEV: 'yes' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TQDM_DISABLE: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' + # Piecewise graph capture and autotuning took 22 minutes at TP4 c5; larger + # batch ladders and TP8 outlast srt-slurm's 1800 s default. + health_check: + max_attempts: 1440 + interval_seconds: 5 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/MiniMax-M3-NVFP4 + # trtllm-serve exposes Prometheus text at /prometheus/metrics. + AIPERF_SERVER_METRICS_URLS: http://localhost:8000/prometheus/metrics + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + KV_OFFLOADING: dram + +# One variant per point. The batch limit is CONC with graphs captured at every +# size to 20 and a thinned ladder at 25 and 30. The host pool is 288 GiB at TP8 +# and about 362 GiB below. +override_tp2_c20: + roles: + agg: + gpus: 2 + args: + tensor_parallel_size: 2 + max_batch_size: 20 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '20' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c25: + roles: + agg: + gpus: 2 + args: + tensor_parallel_size: 2 + max_batch_size: 25 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 17 + - 19 + - 21 + - 23 + - 25 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '25' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c30: + roles: + agg: + gpus: 2 + args: + tensor_parallel_size: 2 + max_batch_size: 30 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 27 + - 30 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '30' + TOTAL_CPU_DRAM_GB: '599' + +override_tp4_c5: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 5 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '5' + TOTAL_CPU_DRAM_GB: '1199' + +override_tp4_c10: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 10 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '10' + TOTAL_CPU_DRAM_GB: '1199' + +override_tp4_c15: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 15 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '15' + TOTAL_CPU_DRAM_GB: '1199' + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 20 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '20' + TOTAL_CPU_DRAM_GB: '1199' + +override_tp4_c25: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 25 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 17 + - 19 + - 21 + - 23 + - 25 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '25' + TOTAL_CPU_DRAM_GB: '1199' + +override_tp4_c30: + roles: + agg: + gpus: 4 + args: + tensor_parallel_size: 4 + max_batch_size: 30 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 27 + - 30 + kv_cache_config: + host_cache_size: 388554555392 + benchmark: + env: + CONC: '30' + TOTAL_CPU_DRAM_GB: '1199' + +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor_parallel_size: 8 + max_batch_size: 1 + cuda_graph_config: + batch_sizes: + - 1 + kv_cache_config: + host_cache_size: 309237645312 + benchmark: + env: + CONC: '1' + TOTAL_CPU_DRAM_GB: '2399' + +override_tp8_c5: + roles: + agg: + gpus: 8 + args: + tensor_parallel_size: 8 + max_batch_size: 5 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + kv_cache_config: + host_cache_size: 309237645312 + benchmark: + env: + CONC: '5' + TOTAL_CPU_DRAM_GB: '2399' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..901ea87ea6 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,264 @@ +# MiniMax-M3 NVFP4 AgentX on B200 with vLLM EAGLE3 (the GQA draft head). +# DRAM KV offload uses SimpleCPUOffloadConnector in lazy mode. +base: + schema: 2 + name: minimaxm3-fp4-b200-vllm-agentic + model: + path: hf:nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: nvidia/MiniMax-M3-NVFP4 + gpu-memory-utilization: 0.9 + block-size: 128 + language-model-only: true + enable-prefix-caching: true + no-enable-flashinfer-autotune: true + reasoning-parser: minimax_m3 + tool-call-parser: minimax_m3 + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + kv-cache-dtype: fp8 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + trust-remote-code: true + # Three-token EAGLE3 with the GQA draft head. Throughput runs switch to + # synthetic rejection at the golden acceptance length. + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + env: + PYTHONNOUSERSITE: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/MiniMax-M3-NVFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. DRAM points give SimpleCPUOffload the whole host budget +# (TOTAL_CPU_DRAM_GB GiB) with lazy offload. +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c5: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '5' + KV_OFFLOADING: none + +override_tp4_c10: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '10' + KV_OFFLOADING: none + +override_tp4_c15: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '15' + KV_OFFLOADING: none + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: none + +override_tp4_c15_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '15' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c20_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '20' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c25_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '25' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c30_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '30' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c32_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c34_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '34' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c36_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '36' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c38_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '38' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' + +override_tp4_c40_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '40' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1082' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..d775524820 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,159 @@ +# MiniMax-M3 NVFP4 AgentX on B300 with vLLM EAGLE3 (the GQA draft head). +# DRAM KV offload uses SimpleCPUOffloadConnector in lazy mode. +base: + schema: 2 + name: minimaxm3-fp4-b300-vllm-agentic + model: + path: hf:nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: nvidia/MiniMax-M3-NVFP4 + gpu-memory-utilization: 0.9 + block-size: 128 + language-model-only: true + enable-prefix-caching: true + no-enable-flashinfer-autotune: true + reasoning-parser: minimax_m3 + tool-call-parser: minimax_m3 + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + kv-cache-dtype: fp8 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + trust-remote-code: true + # Three-token EAGLE3 with the GQA draft head. Throughput runs switch to + # synthetic rejection at the golden acceptance length. + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + env: + PYTHONNOUSERSITE: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/MiniMax-M3-NVFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +# One variant per point. DRAM points give SimpleCPUOffload the whole host budget +# (TOTAL_CPU_DRAM_GB GiB) with lazy offload. +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c5: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '5' + KV_OFFLOADING: none + +override_tp4_c10: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '10' + KV_OFFLOADING: none + +override_tp4_c15: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '15' + KV_OFFLOADING: none + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: none + +override_tp4_c30_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1099511627776,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '30' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1024' + +override_tp2_c24_dram: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":804232626176,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '24' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '749' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..f30dcd6a43 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml @@ -0,0 +1,210 @@ +# MiniMax-M3 MXFP8 AgentX on H100 with vLLM EAGLE3 (the GQA draft head) and +# optional Mooncake DRAM KV offload. 26 GiB of weights per GPU live in host memory. +base: + schema: 2 + name: minimaxm3-fp8-h100-vllm-agentic + model: + path: hf:MiniMaxAI/MiniMax-M3-MXFP8 + container: vllm/vllm-openai:v0.27.1 + precision: fp8 + resources: + gpu_type: h100 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + # Lazy loads of the 31 shards from shared NFS ran at ~105 s/shard, past the + # script's 3600 s, so both windows are two hours. + health_check: + interval_seconds: 10 + max_attempts: 720 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: MiniMaxAI/MiniMax-M3-MXFP8 + tensor-parallel-size: 8 + data-parallel-size: 1 + gpu-memory-utilization: 0.90 + cpu-offload-gb: 26 + attention-backend: TRITON_ATTN + safetensors-load-strategy: lazy + kv-cache-dtype: fp8 + block-size: 128 + language-model-only: true + enable-prefix-caching: true + enable-prompt-tokens-details: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + # Three-token EAGLE3 with the GQA draft head. Throughput runs switch to + # synthetic rejection at the golden acceptance length. + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tool-call-parser: minimax_m3 + reasoning-parser: minimax_m3 + enable-auto-tool-choice: true + trust-remote-code: true + env: + PYTHONNOUSERSITE: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: MiniMaxAI/MiniMax-M3-MXFP8 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + +# One variant per point. The scheduler batch is 2x CONC and graph capture covers +# it at four tokens per sequence (three drafts). +override_tp8_c1: + roles: + agg: + args: + max-num-seqs: 2 + max-cudagraph-capture-size: 8 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c2: + roles: + agg: + args: + max-num-seqs: 4 + max-cudagraph-capture-size: 16 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + +override_tp8_c3: + roles: + agg: + args: + max-num-seqs: 6 + max-cudagraph-capture-size: 24 + benchmark: + env: + CONC: '3' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + args: + max-num-seqs: 8 + max-cudagraph-capture-size: 32 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c5: + roles: + agg: + args: + max-num-seqs: 10 + max-cudagraph-capture-size: 40 + benchmark: + env: + CONC: '5' + KV_OFFLOADING: none + +# DRAM points offload KV to an embedded Mooncake store. Per rank: the host budget +# (1731 GB = 1612 GiB) less the 414 GiB checkpoint page cache, over TP8, less the +# 26 GiB CPU weight offload and the 4 GiB local buffer = 119 GB. +override_tp8_c6_dram: + # The worker's Mooncake client and the master run the same pinned release. + setup_script: vllm-mooncake-0.3.11.sh + services: + - name: mooncake-master + type: mooncake-master + # The in-container install outlasts the default 120 s readiness window. + readiness: + port: 8700 + timeout_seconds: 900 + preamble: >- + pip_install=(python3 -m pip install); + python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages && pip_install+=(--break-system-packages); + "${pip_install[@]}" --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 + args: + - '--eviction_high_watermark_ratio=0.80' + - '--eviction_ratio=0.10' + options: + store_config: + mode: embedded + metadata_server: P2PHANDSHAKE + global_segment_size: 119GB + local_buffer_size: 4GB + protocol: rdma + device_name: '' + enable_offload: false + roles: + agg: + args: + max-num-seqs: 12 + max-cudagraph-capture-size: 48 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' + env: + PYTHONHASHSEED: '0' + MC_SLICE_SIZE: '1048576' + MC_WORKERS_PER_CTX: '4' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + benchmark: + env: + CONC: '6' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_c8_dram: + # The worker's Mooncake client and the master run the same pinned release. + setup_script: vllm-mooncake-0.3.11.sh + services: + - name: mooncake-master + type: mooncake-master + # The in-container install outlasts the default 120 s readiness window. + readiness: + port: 8700 + timeout_seconds: 900 + preamble: >- + pip_install=(python3 -m pip install); + python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages && pip_install+=(--break-system-packages); + "${pip_install[@]}" --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 + args: + - '--eviction_high_watermark_ratio=0.80' + - '--eviction_ratio=0.10' + options: + store_config: + mode: embedded + metadata_server: P2PHANDSHAKE + global_segment_size: 119GB + local_buffer_size: 4GB + protocol: rdma + device_name: '' + enable_offload: false + roles: + agg: + args: + max-num-seqs: 16 + max-cudagraph-capture-size: 64 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' + env: + PYTHONHASHSEED: '0' + MC_SLICE_SIZE: '1048576' + MC_WORKERS_PER_CTX: '4' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + benchmark: + env: + CONC: '8' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..36451e353f --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml @@ -0,0 +1,184 @@ +# MiniMax-M3 MXFP8 AgentX on H200 with vLLM EAGLE3 (the GQA draft head) and +# optional Mooncake DRAM KV offload. +base: + schema: 2 + name: minimaxm3-fp8-h200-vllm-agentic + model: + path: hf:MiniMaxAI/MiniMax-M3-MXFP8 + container: vllm/vllm-openai:nightly-d9105ea8001e0a6d77a96327d17515bb5791fb36 + precision: fp8 + resources: + gpu_type: h200 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: MiniMaxAI/MiniMax-M3-MXFP8 + tensor-parallel-size: 8 + data-parallel-size: 1 + attention-backend: FLASH_ATTN + # Humming for dense MXFP8 linear layers. + linear-backend: humming + kv-cache-dtype: fp8 + block-size: 128 + language-model-only: true + enable-prefix-caching: true + enable-prompt-tokens-details: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + # Three-token EAGLE3 with the GQA draft head. Throughput runs switch to + # synthetic rejection at the golden acceptance length. + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tool-call-parser: minimax_m3 + reasoning-parser: minimax_m3 + enable-auto-tool-choice: true + trust-remote-code: true + env: + PYTHONNOUSERSITE: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + # Atomic reduction for the Marlin MXFP8 MoE path. + VLLM_MARLIN_USE_ATOMIC_ADD: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: MiniMaxAI/MiniMax-M3-MXFP8 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + +# One variant per point. The scheduler batch is 2x CONC and graph capture covers +# it at four tokens per sequence (three drafts). +override_tp8_c1: + roles: + agg: + args: + max-num-seqs: 2 + max-cudagraph-capture-size: 8 + gpu-memory-utilization: 0.95 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c2: + roles: + agg: + args: + max-num-seqs: 4 + max-cudagraph-capture-size: 16 + gpu-memory-utilization: 0.95 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + args: + max-num-seqs: 8 + max-cudagraph-capture-size: 32 + gpu-memory-utilization: 0.95 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c6: + roles: + agg: + args: + max-num-seqs: 12 + max-cudagraph-capture-size: 48 + gpu-memory-utilization: 0.95 + benchmark: + env: + CONC: '6' + KV_OFFLOADING: none + +override_tp8_c8: + roles: + agg: + args: + max-num-seqs: 16 + max-cudagraph-capture-size: 64 + gpu-memory-utilization: 0.95 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp8_c10: + roles: + agg: + args: + max-num-seqs: 20 + max-cudagraph-capture-size: 80 + gpu-memory-utilization: 0.95 + benchmark: + env: + CONC: '10' + KV_OFFLOADING: none + +# The DRAM point offloads KV to an embedded Mooncake store, leaving HBM headroom +# at 0.90. Per rank: the host budget (1234 GB = 1149 GiB) less the 414 GiB +# checkpoint page cache, over TP8, less the 4 GiB local buffer = 87 GB. +override_tp8_c12_dram: + # The worker's Mooncake client and the master run the same pinned release. + setup_script: vllm-mooncake-0.3.11.sh + services: + - name: mooncake-master + type: mooncake-master + # The in-container install outlasts the default 120 s readiness window. + readiness: + port: 8700 + timeout_seconds: 900 + preamble: >- + pip_install=(python3 -m pip install); + python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages && pip_install+=(--break-system-packages); + "${pip_install[@]}" --no-cache-dir --no-deps --force-reinstall mooncake-transfer-engine-cuda13==0.3.11.post1 + args: + - '--eviction_high_watermark_ratio=0.80' + - '--eviction_ratio=0.10' + options: + store_config: + mode: embedded + metadata_server: P2PHANDSHAKE + global_segment_size: 87GB + local_buffer_size: 4GB + protocol: rdma + device_name: '' + enable_offload: false + roles: + agg: + args: + gpu-memory-utilization: 0.9 + max-num-seqs: 24 + max-cudagraph-capture-size: 96 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' + env: + PYTHONHASHSEED: '0' + MC_SLICE_SIZE: '1048576' + MC_WORKERS_PER_CTX: '4' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + benchmark: + env: + CONC: '12' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1234' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..151ce47cc7 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml @@ -0,0 +1,140 @@ +# MiniMax-M3 MXFP8 AgentX on MI300X with vLLM EAGLE3 (GQA draft). The KV cache +# is GPU-resident, or backed by LMCache MP DRAM servers at the offload point. +base: + schema: 2 + name: minimaxm3-fp8-mi300x-vllm-agentic + model: + path: hf:MiniMaxAI/MiniMax-M3-MXFP8 + container: vllm/vllm-openai-rocm:v0.29.0 + precision: fp8 + resources: + gpu_type: mi300x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + # Starts the LMCache servers for the variant that sets LMCACHE_SHARDS. + setup_script: lmcache-mp-rocm.sh + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: MiniMaxAI/MiniMax-M3-MXFP8 + tensor-parallel-size: 8 + trust-remote-code: true + block-size: 128 + gpu-memory-utilization: 0.90 + # MiniMaxM3SparseForConditionalGeneration is not torch-compiled, so + # without breakable graphs the default FULL_AND_PIECEWISE mode aborts. + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + enable-chunked-prefill: true + max-num-batched-tokens: 16384 + language-model-only: true + enable-prefix-caching: true + attention-backend: TRITON_ATTN + kv-cache-dtype: fp8 + tool-call-parser: minimax_m3 + reasoning-parser: minimax_m3 + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + stream-interval: 20 + # Three-token EAGLE3; throughput runs replace verification with the + # golden acceptance length. + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"TRITON_ATTN"}' + env: + PYTHONNOUSERSITE: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + VLLM_ROCM_USE_AITER: '1' + VLLM_ROCM_USE_AITER_MHA: '0' + TORCH_BLAS_PREFER_HIPBLASLT: '1' + NCCL_MIN_NCHANNELS: '112' + GPU_MAX_HW_QUEUES: '2' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: MiniMaxAI/MiniMax-M3-MXFP8 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. Admission is 2x CONC. The DRAM point splits its +# 1298 GB budget into one LMCache L1 shard per TP rank (1298 / 8 = 162 GB). +override_tp8_c2: + roles: + agg: + args: + max-num-seqs: 4 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: 'none' + +override_tp8_c4: + roles: + agg: + args: + max-num-seqs: 8 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: 'none' + +override_tp8_c6: + roles: + agg: + args: + max-num-seqs: 12 + benchmark: + env: + CONC: '6' + KV_OFFLOADING: 'none' + +override_tp8_c8: + roles: + agg: + args: + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: 'none' + +override_tp8_c10: + roles: + agg: + args: + max-num-seqs: 20 + benchmark: + env: + CONC: '10' + KV_OFFLOADING: 'none' + +override_tp8_c16_lmcache: + roles: + agg: + args: + max-num-seqs: 32 + kv-transfer-config: '{"kv_connector":"LMCacheMPConnector","kv_connector_module_path":"lmcache.integration.vllm.lmcache_mp_connector","kv_role":"kv_both","kv_connector_extra_config":{"lmcache.mp.server_urls":"tcp://127.0.0.1:5555,tcp://127.0.0.1:5556,tcp://127.0.0.1:5557,tcp://127.0.0.1:5558,tcp://127.0.0.1:5559,tcp://127.0.0.1:5560,tcp://127.0.0.1:5561,tcp://127.0.0.1:5562","lmcache.mp.mq_timeout":6000.0}}' + env: + LMCACHE_SHARDS: '8' + LMCACHE_L1_SHARD_GB: '162' + benchmark: + env: + CONC: '16' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '1298' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..43e25d9eec --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml @@ -0,0 +1,145 @@ +# MiniMax-M3 MXFP8 AgentX on MI325X with vLLM EAGLE3 (GQA draft). The KV cache +# is GPU-resident. +base: + schema: 2 + name: minimaxm3-fp8-mi325x-vllm-agentic + model: + path: hf:MiniMaxAI/MiniMax-M3-MXFP8 + container: vllm/vllm-openai-rocm:v0.27.1 + precision: fp8 + resources: + gpu_type: mi325x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: MiniMaxAI/MiniMax-M3-MXFP8 + tensor-parallel-size: 8 + gpu-memory-utilization: 0.90 + kv-cache-dtype: fp8 + block-size: 128 + language-model-only: true + attention-backend: TRITON_ATTN + enable-prefix-caching: true + enable-chunked-prefill: true + max-num-batched-tokens: 32768 + # Three-token EAGLE3; throughput runs replace verification with the + # golden acceptance length. + speculative-config: '{"method": "eagle3", "model": "Inferact/MiniMax-M3-EAGLE3-GQA", "num_speculative_tokens": 3, "attention_backend": "TRITON_ATTN"}' + tool-call-parser: minimax_m3 + reasoning-parser: minimax_m3 + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + trust-remote-code: true + stream-interval: 20 + env: + PYTHONNOUSERSITE: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: MiniMaxAI/MiniMax-M3-MXFP8 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. Admission is 2x CONC. +override_tp8_c1: + roles: + agg: + args: + max-num-seqs: 2 + benchmark: + env: + CONC: '1' + +override_tp8_c2: + roles: + agg: + args: + max-num-seqs: 4 + benchmark: + env: + CONC: '2' + +override_tp8_c4: + roles: + agg: + args: + max-num-seqs: 8 + benchmark: + env: + CONC: '4' + +override_tp8_c8: + roles: + agg: + args: + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp8_c10: + roles: + agg: + args: + max-num-seqs: 20 + benchmark: + env: + CONC: '10' + +override_tp8_c12: + roles: + agg: + args: + max-num-seqs: 24 + benchmark: + env: + CONC: '12' + +override_tp8_c14: + roles: + agg: + args: + max-num-seqs: 28 + benchmark: + env: + CONC: '14' + +override_tp8_c16: + roles: + agg: + args: + max-num-seqs: 32 + benchmark: + env: + CONC: '16' + +override_tp8_c18: + roles: + agg: + args: + max-num-seqs: 36 + benchmark: + env: + CONC: '18' diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..d9c5e537a7 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,204 @@ +# MiniMax-M3 MXFP4 AgentX on MI355X with vLLM EAGLE3 (GQA draft). The KV cache +# is GPU-resident. +base: + schema: 2 + name: minimaxm3-fp4-mi355x-vllm-agentic + model: + path: hf:amd/MiniMax-M3-MXFP4 + container: vllm/vllm-openai-rocm:nightly-2a02f6efe319c885e3ccbcecde402e0028f9ec1e + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: amd/MiniMax-M3-MXFP4 + trust-remote-code: true + block-size: 128 + gpu-memory-utilization: 0.90 + enable-chunked-prefill: true + max-num-batched-tokens: 32768 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY", "cudagraph_capture_sizes": [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,22,24,26,28,30,32,34,36,48,64,72,80,88,96,104,112,120,128]}' + language-model-only: true + enable-prefix-caching: true + attention-backend: ROCM_AITER_UNIFIED_ATTN + moe-backend: aiter + kv-cache-dtype: fp8 + attention-config: '{"indexer_kv_dtype": "fp8"}' + tool-call-parser: minimax_m3 + reasoning-parser: minimax_m3 + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + stream-interval: 20 + # Three-token EAGLE3; throughput runs replace verification with the + # golden acceptance length. + speculative-config: '{"method": "eagle3", "model": "Inferact/MiniMax-M3-EAGLE3-GQA", "num_speculative_tokens": 3, "attention_backend": "ROCM_AITER_UNIFIED_ATTN"}' + env: + PYTHONNOUSERSITE: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_USE_BREAKABLE_CUDAGRAPH: '0' + VLLM_ROCM_USE_AITER: '1' + VLLM_ROCM_USE_AITER_MOE: '1' + VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: '1' + VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT: '1' + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: INT4 + VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16: '0' + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB: '256' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: amd/MiniMax-M3-MXFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. Admission is 2x CONC. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 2 + benchmark: + env: + CONC: '1' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 8 + benchmark: + env: + CONC: '4' + +override_tp4_c5: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 10 + benchmark: + env: + CONC: '5' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 16 + benchmark: + env: + CONC: '8' + +override_tp4_c10: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 20 + benchmark: + env: + CONC: '10' + +override_tp4_c12: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 24 + benchmark: + env: + CONC: '12' + +override_tp4_c15: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 30 + benchmark: + env: + CONC: '15' + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 40 + benchmark: + env: + CONC: '20' + +override_tp4_c24: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 48 + benchmark: + env: + CONC: '24' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 64 + benchmark: + env: + CONC: '32' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-num-seqs: 2 + benchmark: + env: + CONC: '1' + +override_tp2_c2: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-num-seqs: 4 + benchmark: + env: + CONC: '2' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..f31bfa382b --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,591 @@ +# Qwen3.5-397B-A17B NVFP4 AgentX on B200 with SGLang native NEXTN MTP. The KV +# cache is GPU-resident or extends to host DRAM through HiCache. +base: + schema: 2 + name: qwen3.5-fp4-b200-sglang-agentic + model: + path: hf:nvidia/Qwen3.5-397B-A17B-NVFP4 + container: lmsysorg/sglang:v0.5.16-cu130 + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + trust-remote-code: true + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.8 + stream-interval: 50 + scheduler-recv-interval: 10 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-metrics: true + enable-cache-report: true + env: + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + # Outlast AIPerf's 300 s client connection pool. + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/Qwen3.5-397B-A17B-NVFP4 + # 256k-capped corpus, within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC and the graph batch is CONC, +# capped at 64. TP4 tokenizes in parallel to keep 256k warmups within the +# client timeout; TP2 keeps the single tokenizer, whose multi-worker startup +# races with TP2 HiCache shared-memory initialization. HiCache sizes each of +# the target KV and Mamba host pools per rank with a 1 GB/rank alignment +# reserve and the 1/15 NEXTN draft pool: (DRAM - TP) * 15 / TP / 31 GB. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp4_c12: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: none + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 32 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 40 + cuda-graph-max-bs: 20 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: none + +override_tp4_c24: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 48 + cuda-graph-max-bs: 24 + benchmark: + env: + CONC: '24' + KV_OFFLOADING: none + +override_tp4_c28: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 56 + cuda-graph-max-bs: 28 + benchmark: + env: + CONC: '28' + KV_OFFLOADING: none + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 64 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: none + +override_tp4_c40: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 80 + cuda-graph-max-bs: 40 + benchmark: + env: + CONC: '40' + KV_OFFLOADING: none + +override_tp4_c48: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 96 + cuda-graph-max-bs: 48 + benchmark: + env: + CONC: '48' + KV_OFFLOADING: none + +override_tp4_c56: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 112 + cuda-graph-max-bs: 56 + benchmark: + env: + CONC: '56' + KV_OFFLOADING: none + +override_tp4_c60: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 120 + cuda-graph-max-bs: 60 + benchmark: + env: + CONC: '60' + KV_OFFLOADING: none + +override_tp4_c62: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 124 + cuda-graph-max-bs: 62 + benchmark: + env: + CONC: '62' + KV_OFFLOADING: none + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 128 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '64' + KV_OFFLOADING: none + +override_tp4_hicache_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 128 + cuda-graph-max-bs: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '64' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c66: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 132 + cuda-graph-max-bs: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '66' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c68: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 136 + cuda-graph-max-bs: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '68' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c70: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 140 + cuda-graph-max-bs: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '70' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c72: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 144 + cuda-graph-max-bs: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '72' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp2_c12: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: none + +override_tp2_c14: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 28 + cuda-graph-max-bs: 14 + benchmark: + env: + CONC: '14' + KV_OFFLOADING: none + +override_tp2_hicache_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 32 + cuda-graph-max-bs: 16 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '16' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '432' + +override_tp2_hicache_c18: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 36 + cuda-graph-max-bs: 18 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '18' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '432' + +override_tp2_hicache_c20: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 40 + cuda-graph-max-bs: 20 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '20' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '432' + +override_tp2_hicache_c22: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 44 + cuda-graph-max-bs: 22 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '22' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '432' + +override_tp2_hicache_c24: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 48 + cuda-graph-max-bs: 24 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '24' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '432' + +override_tp2_hicache_c28: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 56 + cuda-graph-max-bs: 28 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '28' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '432' + +override_tp2_hicache_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 64 + cuda-graph-max-bs: 32 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '432' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..817cebae91 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml @@ -0,0 +1,590 @@ +# Qwen3.5-397B-A17B FP8 AgentX on B200 with SGLang native NEXTN MTP. The KV +# cache is GPU-resident or extends to host DRAM through HiCache. +base: + schema: 2 + name: qwen3.5-fp8-b200-sglang-agentic + model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + precision: fp8 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.8 + stream-interval: 50 + scheduler-recv-interval: 10 + # Parallel tokenization keeps 256k warmups within the client timeout. + tokenizer-worker-num: 6 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-metrics: true + enable-cache-report: true + env: + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + # Outlast AIPerf's 300 s client connection pool. + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + # 256k-capped corpus, within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. Admission is 2x CONC and the decode graph batch is +# CONC, capped at 64. HiCache sizes each of the target KV and Mamba host pools +# per rank with a 1 GB/rank alignment reserve and the 1/15 NEXTN draft pool: +# (DRAM - TP) * 15 / TP / 31 GB. +override_tp8_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 2 + cuda-graph-max-bs-decode: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 8 + cuda-graph-max-bs-decode: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp8_c8: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 16 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp8_c12: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 24 + cuda-graph-max-bs-decode: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: none + +override_tp8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 32 + cuda-graph-max-bs-decode: 16 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp8_c20: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 40 + cuda-graph-max-bs-decode: 20 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: none + +override_tp8_c24: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 48 + cuda-graph-max-bs-decode: 24 + benchmark: + env: + CONC: '24' + KV_OFFLOADING: none + +override_tp8_c28: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 56 + cuda-graph-max-bs-decode: 28 + benchmark: + env: + CONC: '28' + KV_OFFLOADING: none + +override_tp8_c32: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 64 + cuda-graph-max-bs-decode: 32 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: none + +override_tp8_c40: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 80 + cuda-graph-max-bs-decode: 40 + benchmark: + env: + CONC: '40' + KV_OFFLOADING: none + +override_tp8_c48: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 96 + cuda-graph-max-bs-decode: 48 + benchmark: + env: + CONC: '48' + KV_OFFLOADING: none + +override_tp8_c56: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 112 + cuda-graph-max-bs-decode: 56 + benchmark: + env: + CONC: '56' + KV_OFFLOADING: none + +override_tp8_c60: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 120 + cuda-graph-max-bs-decode: 60 + benchmark: + env: + CONC: '60' + KV_OFFLOADING: none + +override_tp8_c62: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 124 + cuda-graph-max-bs-decode: 62 + benchmark: + env: + CONC: '62' + KV_OFFLOADING: none + +override_tp8_c64: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 128 + cuda-graph-max-bs-decode: 64 + benchmark: + env: + CONC: '64' + KV_OFFLOADING: none + +override_tp8_hicache_c64: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 128 + cuda-graph-max-bs-decode: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '64' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_hicache_c66: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 132 + cuda-graph-max-bs-decode: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '66' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_hicache_c68: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 136 + cuda-graph-max-bs-decode: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '68' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_hicache_c70: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 140 + cuda-graph-max-bs-decode: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '70' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_hicache_c72: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 144 + cuda-graph-max-bs-decode: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '72' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp8_hicache_c76: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + max-running-requests: 152 + cuda-graph-max-bs-decode: 64 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '76' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' + +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 2 + cuda-graph-max-bs-decode: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 8 + cuda-graph-max-bs-decode: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 16 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp4_c12: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 24 + cuda-graph-max-bs-decode: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: none + +override_tp4_c14: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 28 + cuda-graph-max-bs-decode: 14 + benchmark: + env: + CONC: '14' + KV_OFFLOADING: none + +override_tp4_hicache_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 32 + cuda-graph-max-bs-decode: 16 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '16' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c18: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 36 + cuda-graph-max-bs-decode: 18 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '18' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c20: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 40 + cuda-graph-max-bs-decode: 20 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '20' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c22: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 44 + cuda-graph-max-bs-decode: 22 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '22' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c24: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 48 + cuda-graph-max-bs-decode: 24 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '24' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c28: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 56 + cuda-graph-max-bs-decode: 28 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '28' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' + +override_tp4_hicache_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 64 + cuda-graph-max-bs-decode: 32 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 104 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '865' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..0c2d2370cd --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,499 @@ +# Qwen3.5-397B-A17B NVFP4 AgentX on B300 with SGLang native NEXTN MTP. +base: + schema: 2 + name: qwen3.5-fp4-b300-sglang-agentic + model: + path: hf:nvidia/Qwen3.5-397B-A17B-NVFP4 + container: lmsysorg/sglang:v0.5.16-cu130 + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + trust-remote-code: true + data-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.8 + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-path: nvidia/Qwen3.5-397B-A17B-NVFP4 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-metrics: true + enable-cache-report: true + env: + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + # Outlast AIPerf's 300 s client connection pool. + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/Qwen3.5-397B-A17B-NVFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + # 256k-capped corpus, within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + +# One variant per point. Admission is 2x CONC and the graph batch is CONC, +# capped at 64. TP4 tokenizes with six workers to keep 256k warmups inside the +# client timeout; TP2 keeps SGLang's single tokenizer, which avoids a startup +# race with HiCache shared-memory initialization. HiCache sizes the target KV +# and Mamba host pools independently, and the NEXTN draft adds 1/15 of the +# target: (599 GB - 1 GB/rank) * 15 / TP2 / 31 = 144 GB per pool. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: 'none' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: 'none' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: 'none' + +override_tp4_c12: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: 'none' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 32 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: 'none' + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 40 + cuda-graph-max-bs: 20 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: 'none' + +override_tp4_c24: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 48 + cuda-graph-max-bs: 24 + benchmark: + env: + CONC: '24' + KV_OFFLOADING: 'none' + +override_tp4_c28: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 56 + cuda-graph-max-bs: 28 + benchmark: + env: + CONC: '28' + KV_OFFLOADING: 'none' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 64 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: 'none' + +override_tp4_c40: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 80 + cuda-graph-max-bs: 40 + benchmark: + env: + CONC: '40' + KV_OFFLOADING: 'none' + +override_tp4_c48: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 96 + cuda-graph-max-bs: 48 + benchmark: + env: + CONC: '48' + KV_OFFLOADING: 'none' + +override_tp4_c56: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 112 + cuda-graph-max-bs: 56 + benchmark: + env: + CONC: '56' + KV_OFFLOADING: 'none' + +override_tp4_c60: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 120 + cuda-graph-max-bs: 60 + benchmark: + env: + CONC: '60' + KV_OFFLOADING: 'none' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 128 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '64' + KV_OFFLOADING: 'none' + +override_tp4_c68: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 136 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '68' + KV_OFFLOADING: 'none' + +override_tp4_c72: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 144 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '72' + KV_OFFLOADING: 'none' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: 'none' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: 'none' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: 'none' + +override_tp2_c12: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: 'none' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 32 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: 'none' + +override_tp2_c20: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 40 + cuda-graph-max-bs: 20 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: 'none' + +override_tp2_c24: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 48 + cuda-graph-max-bs: 24 + benchmark: + env: + CONC: '24' + KV_OFFLOADING: 'none' + +override_tp2_c28: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 56 + cuda-graph-max-bs: 28 + benchmark: + env: + CONC: '28' + KV_OFFLOADING: 'none' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 64 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: 'none' + +override_tp2_c36_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 72 + cuda-graph-max-bs: 36 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '36' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c44_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 88 + cuda-graph-max-bs: 44 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '44' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c52_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 104 + cuda-graph-max-bs: 52 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '52' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..274f545f90 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml @@ -0,0 +1,624 @@ +# Qwen3.5-397B-A17B FP8 AgentX on B300 with SGLang native NEXTN MTP. +base: + schema: 2 + name: qwen3.5-fp8-b300-sglang-agentic + model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang:v0.5.16-cu130 + precision: fp8 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + data-parallel-size: 1 + enable-symm-mem: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.8 + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-path: Qwen/Qwen3.5-397B-A17B-FP8 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-metrics: true + enable-cache-report: true + env: + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + # Outlast AIPerf's 300 s client connection pool. + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + # 256k-capped corpus, within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + +# One variant per point. Admission is 2x CONC and the graph batch is CONC, +# capped at 64. TP4 tokenizes with six workers to keep 256k warmups inside the +# client timeout; TP2 keeps SGLang's single tokenizer, which avoids a startup +# race with HiCache shared-memory initialization. HiCache sizes the target KV +# and Mamba host pools independently, and the NEXTN draft adds 1/15 of the +# target: (599 GB - 1 GB/rank) * 15 / TP2 / 31 = 144 GB per pool. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: 'none' + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: 'none' + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: 'none' + +override_tp4_c12: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: 'none' + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 32 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: 'none' + +override_tp4_c20: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 40 + cuda-graph-max-bs: 20 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: 'none' + +override_tp4_c24: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 48 + cuda-graph-max-bs: 24 + benchmark: + env: + CONC: '24' + KV_OFFLOADING: 'none' + +override_tp4_c28: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 56 + cuda-graph-max-bs: 28 + benchmark: + env: + CONC: '28' + KV_OFFLOADING: 'none' + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 64 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: 'none' + +override_tp4_c40: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 80 + cuda-graph-max-bs: 40 + benchmark: + env: + CONC: '40' + KV_OFFLOADING: 'none' + +override_tp4_c48: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 96 + cuda-graph-max-bs: 48 + benchmark: + env: + CONC: '48' + KV_OFFLOADING: 'none' + +override_tp4_c56: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 112 + cuda-graph-max-bs: 56 + benchmark: + env: + CONC: '56' + KV_OFFLOADING: 'none' + +override_tp4_c60: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 120 + cuda-graph-max-bs: 60 + benchmark: + env: + CONC: '60' + KV_OFFLOADING: 'none' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 128 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '64' + KV_OFFLOADING: 'none' + +override_tp4_c68: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 136 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '68' + KV_OFFLOADING: 'none' + +override_tp4_c72: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + tokenizer-worker-num: 6 + max-running-requests: 144 + cuda-graph-max-bs: 64 + benchmark: + env: + CONC: '72' + KV_OFFLOADING: 'none' + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: 'none' + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: 'none' + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: 'none' + +override_tp2_c12: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: 'none' + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 32 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: 'none' + +override_tp2_c20: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 40 + cuda-graph-max-bs: 20 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: 'none' + +override_tp2_c24: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 48 + cuda-graph-max-bs: 24 + benchmark: + env: + CONC: '24' + KV_OFFLOADING: 'none' + +override_tp2_c28: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 56 + cuda-graph-max-bs: 28 + benchmark: + env: + CONC: '28' + KV_OFFLOADING: 'none' + +override_tp2_c32: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 64 + cuda-graph-max-bs: 32 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: 'none' + +override_tp2_c32_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 64 + cuda-graph-max-bs: 32 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '32' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c34_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 68 + cuda-graph-max-bs: 34 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '34' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c36_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 72 + cuda-graph-max-bs: 36 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '36' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c38_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 76 + cuda-graph-max-bs: 38 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '38' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c40_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 80 + cuda-graph-max-bs: 40 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '40' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c44_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 88 + cuda-graph-max-bs: 44 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '44' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c48_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 96 + cuda-graph-max-bs: 48 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '48' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c52_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 104 + cuda-graph-max-bs: 52 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '52' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' + +override_tp2_c56_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 112 + cuda-graph-max-bs: 56 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 144 + hicache-io-backend: 'kernel' + hicache-mem-layout: 'page_first' + hicache-write-policy: 'write_through_selective' + benchmark: + env: + CONC: '56' + KV_OFFLOADING: 'dram' + TOTAL_CPU_DRAM_GB: '599' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..bc42a7068f --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml @@ -0,0 +1,78 @@ +# Qwen3.5 FP8 AgentX on H100 with SGLang EAGLE MTP; attention is flashinfer +# (trtllm_mha is Blackwell-only). +base: + schema: 2 + name: qwen3.5-fp8-h100-sglang-agentic-mtp + model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 + precision: fp8 + resources: + gpu_type: h100 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: flashinfer + enable-flashinfer-allreduce-fusion: true + # 80 GB HBM3 keeps the static fraction at 0.75. + mem-fraction-static: 0.75 + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + enable-metrics: true + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + env: + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_SPEC_V2: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + # H100 caps max_model_len at 131k; the 256k-capped with-subagents corpus + # keeps the rejection rate far below the unfiltered ~1M-token corpus. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_with_subagents_256k + +override_gpu_kv: + benchmark: + env: + KV_OFFLOADING: none +# HiCache extends RadixAttention. Hybrid GDN/Mamba allocates one KV and one +# Mamba host pool per rank: 1731 GB / TP8 / 2 pools = 108 GB each. +override_hicache: + roles: + agg: + args: + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 108 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml new file mode 100644 index 0000000000..6eeeaeca9d --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml @@ -0,0 +1,75 @@ +# Qwen3.5 FP8 AgentX on H100 with SGLang, no speculative decoding; attention is +# flashinfer (trtllm_mha is Blackwell-only). +base: + schema: 2 + name: qwen3.5-fp8-h100-sglang-agentic + model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang:v0.5.12-cu130 + precision: fp8 + resources: + gpu_type: h100 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + # This release drops cached_tokens_details on the multi-tokenizer path. + setup_script: sglang-multi-tokenizer-cached-tokens-details.sh + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: flashinfer + enable-flashinfer-allreduce-fusion: true + # 80 GB HBM3 keeps the static fraction at 0.75. + mem-fraction-static: 0.75 + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + # H100 caps max_model_len at 131k; the 256k-capped with-subagents corpus + # keeps the rejection rate far below the unfiltered ~1M-token corpus. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_with_subagents_256k + +override_gpu_kv: + benchmark: + env: + KV_OFFLOADING: none +# HiCache extends RadixAttention. Hybrid GDN/Mamba allocates one KV and one +# Mamba host pool per rank: 1731 GB / TP8 / 2 pools = 108 GB each. +override_hicache: + roles: + agg: + args: + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 108 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1731' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..519e6efafc --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml @@ -0,0 +1,95 @@ +# Qwen3.5 FP8 AgentX on H200 with SGLang EAGLE MTP; attention is flashinfer +# (trtllm_mha is Blackwell-only). +base: + schema: 2 + name: qwen3.5-fp8-h200-sglang-agentic + model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9 + precision: fp8 + resources: + gpu_type: h200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + attention-backend: flashinfer + enable-flashinfer-allreduce-fusion: true + mem-fraction-static: 0.8 + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + enable-metrics: true + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + env: + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_SPEC_V2: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + # 256k-capped with-subagents corpus: the unfiltered corpus has requests up + # to ~1M tokens, beyond this model's TP8 context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_with_subagents_256k + +override_gpu_kv: + benchmark: + env: + KV_OFFLOADING: none +# HiCache extends RadixAttention. Hybrid GDN/Mamba allocates one KV and one +# Mamba host pool per rank: 1234 GB / TP8 / 2 pools = 77 GB each. +override_hicache: + roles: + agg: + args: + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 77 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1234' +# The same HiCache arm with the experts TP-sharded (EP1) on a later nightly. +override_hicache_ep1: + model: + container: lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9 + roles: + agg: + args: + expert-parallel-size: 1 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 77 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_through_selective + benchmark: + env: + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1234' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..3c3cded953 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml @@ -0,0 +1,175 @@ +# Qwen3.5 FP8 AgentX on MI300X with SGLang native EAGLE MTP. The KV cache is +# GPU-resident. +base: + schema: 2 + name: qwen3.5-fp8-mi300x-sglang-agentic + model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730 + precision: fp8 + resources: + gpu_type: mi300x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + # Loading plus first-request kernel tuning outlasts the 1800 s default: + # MI300X Qwen3.5 was ready after ~27 minutes (run 36160403972). + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + tokenizer-path: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + data-parallel-size: 1 + attention-backend: aiter + enable-aiter-allreduce-fusion: true + mamba-ssm-dtype: bfloat16 + tokenizer-worker-num: 6 + max-prefill-tokens: 32768 + scheduler-recv-interval: 30 + mem-fraction-static: 0.75 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-metrics: true + enable-cache-report: true + env: + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + # 256k-capped corpus: the unfiltered corpus exceeds this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. AgentX concurrency counts live session trees, so +# admission is 2x CONC; the graph batch is CONC, capped at 64. +override_tp8_ep1_c4: + roles: + agg: + args: + expert-parallel-size: 1 + cuda-graph-max-bs: 4 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + +override_tp8_ep1_c16: + roles: + agg: + args: + expert-parallel-size: 1 + cuda-graph-max-bs: 16 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + +override_tp8_ep1_c20: + roles: + agg: + args: + expert-parallel-size: 1 + cuda-graph-max-bs: 20 + max-running-requests: 40 + benchmark: + env: + CONC: '20' + +override_tp8_ep1_c24: + roles: + agg: + args: + expert-parallel-size: 1 + cuda-graph-max-bs: 24 + max-running-requests: 48 + benchmark: + env: + CONC: '24' + +override_tp8_ep1_c32: + roles: + agg: + args: + expert-parallel-size: 1 + cuda-graph-max-bs: 32 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + +override_tp8_ep8_c4: + roles: + agg: + args: + expert-parallel-size: 8 + cuda-graph-max-bs: 4 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + +override_tp8_ep8_c16: + roles: + agg: + args: + expert-parallel-size: 8 + cuda-graph-max-bs: 16 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + +override_tp8_ep8_c20: + roles: + agg: + args: + expert-parallel-size: 8 + cuda-graph-max-bs: 20 + max-running-requests: 40 + benchmark: + env: + CONC: '20' + +override_tp8_ep8_c24: + roles: + agg: + args: + expert-parallel-size: 8 + cuda-graph-max-bs: 24 + max-running-requests: 48 + benchmark: + env: + CONC: '24' + +override_tp8_ep8_c32: + roles: + agg: + args: + expert-parallel-size: 8 + cuda-graph-max-bs: 32 + max-running-requests: 64 + benchmark: + env: + CONC: '32' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..96e1d7b329 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml @@ -0,0 +1,551 @@ +# Qwen3.5 FP8 AgentX on MI325X with SGLang native EAGLE MTP. The KV cache is +# GPU-resident. +base: + schema: 2 + name: qwen3.5-fp8-mi325x-sglang-agentic + model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang:v0.5.16-rocm720-mi30x + precision: fp8 + resources: + gpu_type: mi325x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + # Loading plus first-request kernel tuning outlasts the 1800 s default: + # MI300X Qwen3.5 was ready after ~27 minutes (run 36160403972). + health_check: + interval_seconds: 10 + max_attempts: 360 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + tokenizer-path: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + data-parallel-size: 1 + attention-backend: aiter + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + model-loader-extra-config: '{"enable_multithread_load": true}' + watchdog-timeout: 1200 + enable-aiter-allreduce-fusion: true + page-size: 16 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + scheduler-recv-interval: 30 + stream-interval: 50 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-metrics: true + enable-cache-report: true + env: + PYTHONNOUSERSITE: '1' + SGLANG_USE_AITER: '1' + SGLANG_USE_AITER_UNIFIED_ATTN: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + # 256k-capped corpus: the unfiltered corpus exceeds this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. AgentX concurrency counts live session trees, so +# admission is 2x CONC; the graph batch is CONC, capped at 64. With EAGLE, +# SGLang reserves 15% of the static budget for the draft model: TP2 needs 0.95 +# (an effective 0.8075) to fit weights, hybrid state and KV; TP4/TP8 use 0.80. +# Tokenizer workers run only from TP4. +override_tp2_ep2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.95 + cuda-graph-max-bs: 1 + max-running-requests: 2 + benchmark: + env: + CONC: '1' + +override_tp2_ep2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.95 + cuda-graph-max-bs: 4 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + +override_tp2_ep2_c6: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + mem-fraction-static: 0.95 + cuda-graph-max-bs: 6 + max-running-requests: 12 + benchmark: + env: + CONC: '6' + +override_tp4_ep1_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 1 + max-running-requests: 2 + benchmark: + env: + CONC: '1' + +override_tp4_ep1_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 8 + max-running-requests: 16 + benchmark: + env: + CONC: '8' + +override_tp4_ep1_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 16 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + +override_tp4_ep1_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 32 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + +override_tp4_ep1_c40: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 40 + max-running-requests: 80 + benchmark: + env: + CONC: '40' + +override_tp4_ep1_c48: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 48 + max-running-requests: 96 + benchmark: + env: + CONC: '48' + +override_tp4_ep4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 4 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + +override_tp4_ep4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 16 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + +override_tp4_ep4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 32 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + +override_tp4_ep4_c40: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 40 + max-running-requests: 80 + benchmark: + env: + CONC: '40' + +override_tp4_ep4_c48: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 48 + max-running-requests: 96 + benchmark: + env: + CONC: '48' + +override_tp8_ep1_c1: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 1 + max-running-requests: 2 + benchmark: + env: + CONC: '1' + +override_tp8_ep1_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 4 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + +override_tp8_ep1_c8: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 8 + max-running-requests: 16 + benchmark: + env: + CONC: '8' + +override_tp8_ep1_c12: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 12 + max-running-requests: 24 + benchmark: + env: + CONC: '12' + +override_tp8_ep1_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 16 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + +override_tp8_ep1_c24: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 24 + max-running-requests: 48 + benchmark: + env: + CONC: '24' + +override_tp8_ep1_c32: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 32 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + +override_tp8_ep1_c40: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 40 + max-running-requests: 80 + benchmark: + env: + CONC: '40' + +override_tp8_ep1_c48: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 48 + max-running-requests: 96 + benchmark: + env: + CONC: '48' + +override_tp8_ep1_c64: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 64 + max-running-requests: 128 + benchmark: + env: + CONC: '64' + +override_tp8_ep1_c80: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 64 + max-running-requests: 160 + benchmark: + env: + CONC: '80' + +override_tp8_ep8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 8 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 4 + max-running-requests: 8 + benchmark: + env: + CONC: '4' + +override_tp8_ep8_c16: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 8 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 16 + max-running-requests: 32 + benchmark: + env: + CONC: '16' + +override_tp8_ep8_c24: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 8 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 24 + max-running-requests: 48 + benchmark: + env: + CONC: '24' + +override_tp8_ep8_c32: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 8 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 32 + max-running-requests: 64 + benchmark: + env: + CONC: '32' + +override_tp8_ep8_c40: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 8 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 40 + max-running-requests: 80 + benchmark: + env: + CONC: '40' + +override_tp8_ep8_c48: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 8 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 48 + max-running-requests: 96 + benchmark: + env: + CONC: '48' + +override_tp8_ep8_c64: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + expert-parallel-size: 8 + mem-fraction-static: 0.8 + tokenizer-worker-num: 6 + cuda-graph-max-bs: 64 + max-running-requests: 128 + benchmark: + env: + CONC: '64' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..749c769773 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,323 @@ +# Qwen3.5-397B-A17B MXFP4 AgentX on MI355X with SGLang native EAGLE MTP; KV is +# GPU-resident or extends to host DRAM through HiCache. +base: + schema: 2 + name: qwen3.5-fp4-mi355x-sglang-agentic + model: + path: hf:amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 + container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 + tokenizer-path: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 + trust-remote-code: true + data-parallel-size: 1 + expert-parallel-size: 1 + attention-backend: aiter + mem-fraction-static: 0.8 + model-loader-extra-config: '{"enable_multithread_load": true}' + watchdog-timeout: 1200 + page-size: 16 + kv-cache-dtype: fp8_e4m3 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + scheduler-recv-interval: 60 + stream-interval: 50 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-metrics: true + enable-cache-report: true + env: + PYTHONNOUSERSITE: '1' + SGLANG_USE_AITER: '1' + SGLANG_USE_AITER_UNIFIED_ATTN: '1' + AITER_FLYDSL_FORCE: '1' + SGLANG_MAMBA_SSM_DTYPE: bfloat16 + ROCM_QUICK_REDUCE_QUANTIZATION: INT4 + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 + # 256k-capped corpus within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +# One variant per point. Admission is 2x CONC and the decode graph batch follows +# it up to 128; TP4 adds tokenizer workers. HiCache holds 1.5x the device KV pool. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp4_c12: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 24 + cuda-graph-max-bs-decode: 24 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: none + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + tokenizer-worker-num: 6 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp2_c1: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp2_c8: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp2_c12: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 24 + cuda-graph-max-bs-decode: 24 + benchmark: + env: + CONC: '12' + KV_OFFLOADING: none + +override_tp2_c16: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp2_c20: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 40 + cuda-graph-max-bs-decode: 40 + benchmark: + env: + CONC: '20' + KV_OFFLOADING: none + +override_tp2_c20_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 40 + cuda-graph-max-bs-decode: 40 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first + benchmark: + env: + CONC: '20' + KV_OFFLOADING: dram + +override_tp2_c24_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 48 + cuda-graph-max-bs-decode: 48 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first + benchmark: + env: + CONC: '24' + KV_OFFLOADING: dram + +override_tp2_c28_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 56 + cuda-graph-max-bs-decode: 56 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first + benchmark: + env: + CONC: '28' + KV_OFFLOADING: dram + +override_tp2_c32_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 64 + cuda-graph-max-bs-decode: 64 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + +override_tp2_c36_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 72 + cuda-graph-max-bs-decode: 72 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first + benchmark: + env: + CONC: '36' + KV_OFFLOADING: dram + +override_tp2_c40_hicache: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-running-requests: 80 + cuda-graph-max-bs-decode: 80 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first + benchmark: + env: + CONC: '40' + KV_OFFLOADING: dram diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..f355e0b994 --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml @@ -0,0 +1,110 @@ +# Qwen3.8-Flash-Next NVFP4 AgentX on B200 via SGLang with native NEXTN MTP. +# The 126 GiB checkpoint fits on one GPU, so the cookbook command is TP1. +base: + schema: 2 + name: qwen3.8next-fp4-b200-sglang-agentic + model: + path: hf:RadixArk/Qwen3.8-Flash-Next-NVFP4 + container: lmsysorg/sglang:qwen38flashnext + precision: fp4 + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 1 + args: + served-model-name: RadixArk/Qwen3.8-Flash-Next-NVFP4 + trust-remote-code: true + tensor-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + # Quantization is read from the checkpoint. + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + # Flashinfer linear-attn decode on SM100+ requires a bfloat16 state. + mamba-ssm-dtype: bfloat16 + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + reasoning-parser: auto + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + enable-metrics: true + enable-cache-report: true + env: + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + # Outlast AIPerf's 300 s client connection pool. + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: RadixArk/Qwen3.8-Flash-Next-NVFP4 + # 256k-capped corpus, within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# One variant per point. NEXTN resets an unset admission cap to 48, so it is +# explicit: 2x CONC, with the graph batch equal to it. C16 needs a larger +# static fraction and a bounded Mamba cache. +override_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 2 + mem-fraction-static: 0.8 + benchmark: + env: + CONC: '1' + +override_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 8 + mem-fraction-static: 0.8 + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 16 + mem-fraction-static: 0.8 + benchmark: + env: + CONC: '8' + +override_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 32 + mem-fraction-static: 0.9 + max-mamba-cache-size: 160 + benchmark: + env: + CONC: '16' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..52879777ed --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml @@ -0,0 +1,121 @@ +# Qwen3.8-Flash-Next NVFP4 AgentX on B300 via SGLang with native NEXTN MTP. +# The 126 GiB checkpoint fits on one B300, so the cookbook command is TP1. +base: + schema: 2 + name: qwen3.8next-fp4-b300-sglang-agentic + model: + path: hf:RadixArk/Qwen3.8-Flash-Next-NVFP4 + container: lmsysorg/sglang:qwen38flashnext + precision: fp4 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 1 + args: + served-model-name: RadixArk/Qwen3.8-Flash-Next-NVFP4 + trust-remote-code: true + tensor-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + # Quantization is read from the checkpoint; the hybrid GDN + # linear-attention layers take their own backends. + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + # SM100+ flashinfer linear-attn decode requires a bf16 SSM state. + mamba-ssm-dtype: bfloat16 + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + reasoning-parser: auto + mem-fraction-static: 0.8 + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-path: RadixArk/Qwen3.8-Flash-Next-NVFP4 + enable-metrics: true + enable-cache-report: true + env: + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + # Outlast AIPerf's 300 s client connection pool. + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + # The checkpoint loads from the shared HF cache on Lustre, which took over + # 30 minutes on a cold node. + health_check: + max_attempts: 1440 + interval_seconds: 5 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: RadixArk/Qwen3.8-Flash-Next-NVFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + # 256k-capped corpus, within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + +# One variant per point: admission is 2x CONC and the graph batch is CONC. NEXTN +# resets an unset admission limit to 48, so it stays explicit. +override_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + +override_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + +override_c12: + roles: + agg: + args: + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + +override_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '16' diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..d780dbe25b --- /dev/null +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml @@ -0,0 +1,105 @@ +# Qwen3.8-Flash-Next FP8 AgentX on H200 via SGLang with native NEXTN MTP. +base: + schema: 2 + name: qwen3.8next-fp8-h200-sglang-agentic + model: + path: hf:Qwen/Qwen3.8-Flash-Next-FP8 + container: lmsysorg/sglang:qwen38flashnext + precision: fp8 + resources: + gpu_type: h200 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + args: + served-model-name: Qwen/Qwen3.8-Flash-Next-FP8 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 4 + data-parallel-size: 1 + mem-fraction-static: 0.85 + chunked-prefill-size: 8192 + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + mamba-ssm-dtype: float32 + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + reasoning-parser: auto + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_SPEC_V2: '1' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: Qwen/Qwen3.8-Flash-Next-FP8 + # 256k-capped with-subagents corpus, within this model's context. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_with_subagents_256k + +# One variant per point: admission is 2x CONC and the graph batch is CONC. +override_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs: 1 + benchmark: + env: + CONC: '1' + +override_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs: 4 + benchmark: + env: + CONC: '4' + +override_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs: 8 + benchmark: + env: + CONC: '8' + +override_c12: + roles: + agg: + args: + max-running-requests: 24 + cuda-graph-max-bs: 12 + benchmark: + env: + CONC: '12' + +override_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 16 + benchmark: + env: + CONC: '16' diff --git a/benchmarks/single_node/srt_eval.sh b/benchmarks/single_node/srt_eval.sh index e6c8af8af4..4fe17dafcb 100644 --- a/benchmarks/single_node/srt_eval.sh +++ b/benchmarks/single_node/srt_eval.sh @@ -10,7 +10,18 @@ SRT_EVAL_STATUS_FILE="$2" trap 'rc=$?; printf "%s\n" "$rc" > "$SRT_EVAL_STATUS_FILE"' EXIT source "$(dirname "${BASH_SOURCE[0]}")/../benchmark_lib.sh" -check_env_vars MODEL MODEL_NAME CONC TP EP_SIZE DP_ATTENTION IS_MULTINODE MAX_MODEL_LEN +check_env_vars MODEL MODEL_NAME CONC TP EP_SIZE DP_ATTENTION IS_MULTINODE IS_AGENTIC +# AgentX evaluates at the native context with the workflow's eval framework. +eval_args=() +if [[ "$IS_AGENTIC" != 1 ]]; then + check_env_vars MAX_MODEL_LEN + eval_args=(--framework lm-eval) +elif [[ "${MODEL_PREFIX:-}" == glm5.2 ]]; then + # GLM-5.2's template defaults to maximum reasoning effort without + # chat_template_kwargs, which mini-swe-agent never passes; the heavy thinking + # exhausts the shared step budget. The recipe env does not reach post-eval. + export SWEBENCH_AGENT_STEP_LIMIT=150 +fi export PORT="${1##*:}" if [[ ! "$PORT" =~ ^[1-9][0-9]*$ || "$IS_MULTINODE" != false ]]; then echo "ERROR: single-node eval requires a local endpoint and single-node metadata" >&2 @@ -22,6 +33,9 @@ if [[ -d /model ]]; then fi eval_rc=0 -run_eval --framework lm-eval --port "$PORT" || eval_rc=$? -append_lm_eval_summary || eval_rc=1 +run_eval "${eval_args[@]}" --port "$PORT" || eval_rc=$? +# AgentX eval-only run_eval already staged and removed its results. +if [[ "$IS_AGENTIC" != 1 ]]; then + append_lm_eval_summary || eval_rc=1 +fi exit "$eval_rc" diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/srt_agentic.sh similarity index 72% rename from benchmarks/multi_node/agentic_srt.sh rename to benchmarks/srt_agentic.sh index 338fcb966b..09aedbc1b9 100644 --- a/benchmarks/multi_node/agentic_srt.sh +++ b/benchmarks/srt_agentic.sh @@ -2,12 +2,20 @@ set -eo pipefail set -x -# Client-only agentic trace replay for srt-slurm multinode jobs. +# Client-only AgentX trace replay for single- and multi-node srt-slurm jobs. # srt-slurm owns server startup; this script runs as benchmark.type=custom -# against the already-ready frontend on the head node. +# against the already-ready frontend. Multi-node batches replay each CONC_LIST +# point in turn; a single-node job replays its one CONC point. -source "$(dirname "${BASH_SOURCE[0]}")/../benchmark_lib.sh" --validation-only -check_env_vars INFMAX_CONTAINER_WORKSPACE RESULT_DIR EVAL_ONLY AIPERF_DRAIN_TIMEOUT_SECONDS AIPERF_DRAIN_POLL_SECONDS +# Jobs inherit the legacy scripts' /workspace, which srt-slurm does not mount; +# fall back to the repo mount this client runs from. +if [[ ! -f "${INFMAX_CONTAINER_WORKSPACE:-}/benchmarks/benchmark_lib.sh" ]]; then + INFMAX_CONTAINER_WORKSPACE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +fi +: "${IS_MULTINODE:=false}" "${PORT:=8000}" +export INFMAX_CONTAINER_WORKSPACE IS_MULTINODE PORT +source "$INFMAX_CONTAINER_WORKSPACE/benchmarks/benchmark_lib.sh" --validation-only +check_env_vars RESULT_DIR EVAL_ONLY source "$INFMAX_CONTAINER_WORKSPACE/benchmarks/benchmark_lib.sh" if [[ -n "${SRT_FRONTEND_HOST:-}" ]]; then @@ -41,9 +49,21 @@ if [[ -z "${AIPERF_SERVER_URL:-}" ]]; then fi echo "Using srt-slurm frontend endpoint: $AIPERF_SERVER_URL" +# A router frontend does not re-export engine metrics; read them from each worker. +if [[ -z "${AIPERF_SERVER_METRICS_URLS:-}" && "${SRTCTL_FRONTEND_TYPE:-}" != dynamo ]]; then + endpoints="${SRT_AGG_ENDPOINTS:-${SRT_PREFILL_ENDPOINTS:+$SRT_PREFILL_ENDPOINTS,}${SRT_DECODE_ENDPOINTS:-}}" + if [[ -n "${endpoints%,}" ]]; then + AIPERF_SERVER_METRICS_URLS=$(sed -E 's#([^,]+)#http://\1/metrics#g' <<< "${endpoints%,}") + export AIPERF_SERVER_METRICS_URLS + fi +fi + BASE_RESULT_DIR="${RESULT_DIR}" BASE_RESULT_FILENAME="$RESULT_FILENAME" read -r -a CONCURRENCIES <<< "${CONC_LIST:-$CONC}" +if (( ${#CONCURRENCIES[@]} > 1 )); then + check_env_vars AIPERF_DRAIN_TIMEOUT_SECONDS AIPERF_DRAIN_POLL_SECONDS +fi if [ "${#CONCURRENCIES[@]}" -eq 0 ]; then echo "ERROR: CONC_LIST must contain at least one concurrency" >&2 @@ -138,13 +158,24 @@ PY for index in "${!CONCURRENCIES[@]}"; do concurrency="${CONCURRENCIES[$index]}" export CONC="$concurrency" - export RESULT_FILENAME="${BASE_RESULT_FILENAME}_conc${concurrency}" - RESULT_DIR="${BASE_RESULT_DIR}/conc_${concurrency}" + # Multi-node collection expects per-point names; a single-node job keeps the workflow's. + if [[ -n "${CONC_LIST:-}" ]]; then + export RESULT_FILENAME="${BASE_RESULT_FILENAME}_conc${concurrency}" + RESULT_DIR="${BASE_RESULT_DIR}/conc_${concurrency}" + fi mkdir -p "$RESULT_DIR" echo "Running agentic concurrency $concurrency of: ${CONCURRENCIES[*]}" build_replay_cmd "$RESULT_DIR" + # Recipes whose legacy launch rendered prompts client-side opt in here. + if [[ "${AIPERF_APPLY_CHAT_TEMPLATE:-}" == true ]]; then + REPLAY_CMD+=" --apply-chat-template" + fi + # Bounded post-window drain for long responses admitted near the end. + if [[ -n "${AIPERF_BENCHMARK_GRACE_PERIOD:-}" ]]; then + REPLAY_CMD+=" --benchmark-grace-period $AIPERF_BENCHMARK_GRACE_PERIOD" + fi run_agentic_replay_and_write_outputs "$RESULT_DIR" if [ "$index" -lt "$(( ${#CONCURRENCIES[@]} - 1 ))" ]; then diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 543c404580..ce2e4e0bc6 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -235,11 +235,11 @@ qwen3.5-fp8-mi325x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 6] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 32, 40, 48] } - - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 32, 40, 48] } - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 24, 32, 40, 48, 64, 80] } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 24, 32, 40, 48, 64] } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 6], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 32, 40, 48], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml } + - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 32, 40, 48], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 24, 32, 40, 48, 64, 80], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 24, 32, 40, 48, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml } qwen3.5-fp8-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 @@ -420,9 +420,9 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 36, 40] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 36, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 @@ -488,8 +488,8 @@ qwen3.5-fp8-mi300x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32] } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32] } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml } dsr1-fp8-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 @@ -664,7 +664,7 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.60 search-space: - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1, 4] , spec-decoding: mtp} + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1, 4] , spec-decoding: mtp } - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 10, 12, 14], spec-decoding: mtp } - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [44, 48, 70], spec-decoding: mtp } @@ -701,7 +701,7 @@ kimik3-fp4-mi355x-atom-agentic-mtp: agentic-coding: - dram-utilization: 0.343 search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 4], spec-decoding: mtp } + - { tp: 8, kv-offloading: none, conc-list: [1, 4], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml } - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [14, 16, 48], spec-decoding: mtp } - dram-utilization: 0.513 search-space: @@ -721,8 +721,8 @@ minimaxm3-fp4-mi355x-atom-agentic-mtp: # = TOTAL_CPU_DRAM_GB / TP = node_DRAM * dram-utilization / 8 (TP-independent). - dram-utilization: 0.687 search-space: - - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 5, 8, 10, 12, 15, 20, 24, 28, 32], spec-decoding: mtp } - - { tp: 2, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp } + - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 5, 8, 10, 12, 15, 20, 24, 28, 32], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml } - { tp: 2, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5rc3+rocm7.2.4" }, conc-list: [20, 25, 30], spec-decoding: mtp } - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5rc3+rocm7.2.4" }, conc-list: [40, 48], spec-decoding: mtp } @@ -904,10 +904,10 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.60 search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32, 40, 48, 56, 64] } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32, 40, 48, 56, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml } # Single-node validation found c128 to be the DPA throughput peak among # c64/c128/c256. Add c192 to locate the knee before the c256 regression. - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64, 128, 192], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64, 128, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml } # LMCache invalid-block recovery currently assumes one KV-cache group, # while speculative decoding creates two. Restore these points after the # upstream hybrid KV recovery fix lands: @@ -927,8 +927,8 @@ dsv4-fp4-mi355x-atom-agentic-mtp: scenarios: agentic-coding: - search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [48, 64, 96, 128, 256] } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [48, 64, 96, 128, 256], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml } dsr1-fp4-mi355x-sglang-disagg-mtp: image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 @@ -1009,8 +1009,8 @@ minimaxm3-fp8-mi300x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 6, 8, 10] } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [16] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 6, 8, 10], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml } # DeepSeek-V4.1-Flash AgentX on MI300X (gfx942), a copy of the validated MI355X vLLM # arm (run 34710937012) on the same ROCm nightly. gfx942 is not in the upstream @@ -1036,10 +1036,10 @@ dsv41flash-fp4-mi300x-vllm-agentic-dspark: # the tables would not fit beside the checkpoint share on a 192 GB card. # vllm-project/vllm#57491 widened the two is_cuda() gates to # is_cuda_alike(), so engram_config resolves on gfx942 in this image. - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml } # TP4 halves the GPUs per server. With the Engram tables in pinned host # memory the resident weights are ~81 GiB per rank. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml } # No TP2 arm on this SKU. Measured in run 35671005506: at TP2 the resident # weights are ~145 GiB per rank, and even with the indexer buffer halved # to 4096 batched tokens the engine reported "Available KV cache memory: @@ -1061,7 +1061,7 @@ glm5.2-fp8-mi325x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 3, 4, 5, 6, 8] } + - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 3, 4, 5, 6, 8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml } minimaxm3-fp8-mi325x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:v0.27.1 @@ -1075,7 +1075,7 @@ minimaxm3-fp8-mi325x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.20 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml } # DeepSeek-V4.1-Flash AgentX on MI325X (gfx942), a copy of the validated MI355X vLLM # arm (run 34710937012) on the same ROCm nightly. gfx942 is not in the upstream @@ -1101,13 +1101,13 @@ dsv41flash-fp4-mi325x-vllm-agentic-dspark: # the tables would not fit beside the checkpoint share on a 256 GB card. # vllm-project/vllm#57491 widened the two is_cuda() gates to # is_cuda_alike(), so engram_config resolves on gfx942 in this image. - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml } # TP4 halves the GPUs per server. With the Engram tables in pinned host # memory the resident weights are ~81 GiB per rank. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml } # TP2 quarters them. Resident weights rise to ~145 GiB per rank, so the # recipe halves the indexer buffer and caps the scheduler here. - - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml } dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260923 @@ -1123,9 +1123,10 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - # All arms use umbp-linker. DSpark gamma 6 at c4/c16 (golden AL peak), 3 above. + # All arms offload to umbp-linker. The recipe carries each point's sizing and + # DSpark draft length (6 at c4/c16, 3 above). - spec-decoding: "draft_model" - conc-list: [ 4 ] + conc-list: [4] kv-offloading: dram kv-offload-backend: { name: umbp-linker } prefill: @@ -1134,18 +1135,14 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907" + - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c4" decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=6" - spec-decoding: "draft_model" - conc-list: [ 16 ] + conc-list: [16] kv-offloading: dram kv-offload-backend: { name: umbp-linker } prefill: @@ -1154,18 +1151,14 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907" + - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c16" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=6" - spec-decoding: "draft_model" - conc-list: [ 32, 48 ] + conc-list: [32] kv-offloading: dram kv-offload-backend: { name: umbp-linker } prefill: @@ -1174,18 +1167,30 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907" + - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c32" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false + - spec-decoding: "draft_model" + conc-list: [48] + kv-offloading: dram + kv-offload-backend: { name: umbp-linker } + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" + - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c48" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false - spec-decoding: "draft_model" - conc-list: [ 192, 256 ] + conc-list: [192] kv-offloading: dram kv-offload-backend: { name: umbp-linker } prefill: @@ -1194,17 +1199,28 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: ep: 1 dp-attn: true additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_ROUTER_POLICY=consistent_hashing" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907" + - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c192" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: true + - spec-decoding: "draft_model" + conc-list: [256] + kv-offloading: dram + kv-offload-backend: { name: umbp-linker } + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" + - "CONFIG_FILE=recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml:override_c256" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: image: rocm/atom-dev:nightly_202609221542 @@ -1325,8 +1341,8 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.85 search-space: - - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 10, 12, 14, 16], spec-decoding: mtp } - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp } + - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 10, 12, 14, 16], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml } # GLM-5.2 FP8 agentic-coding benchmark on MI355X via SGLang with MTP speculative # decoding: the FP8 precision sibling of glm5.2-fp4-mi355x-sglang-agentic-mtp, @@ -1355,8 +1371,8 @@ glm5.2-fp8-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 10], spec-decoding: mtp } - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8, 12, 16], spec-decoding: mtp } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 10], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8, 12, 16], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM with MTP speculative # decoding, matching the standalone recipe (ROCm/ATOM PR 2345). Small # concurrency (C2-C10) is TP4-only on the GPU prefix cache with no KV offload; @@ -1379,9 +1395,9 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: # Only the DCP4 arm offloads; the other two are GPU-resident. - dram-utilization: 0.171 search-space: - - { tp: 4, kv-offloading: none, conc-list: [2, 4, 8, 10], spec-decoding: mtp } + - { tp: 4, kv-offloading: none, conc-list: [2, 4, 8, 10], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml } - { tp: 4, dcp-size: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [16, 24, 32, 40, 48], spec-decoding: mtp } - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp } + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml } dsv4-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260914 @@ -1395,9 +1411,9 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 16], spec-decoding: draft_model } - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 48], spec-decoding: draft_model } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [128, 256], spec-decoding: draft_model } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 16], spec-decoding: draft_model, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 48], spec-decoding: draft_model, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [128, 256], spec-decoding: draft_model, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml } # Upstream ROCm nightly rather than the deepseekv41-flash-0909 release tag: that # tag predates vllm-project/vllm#56503, which moves the mHC delayed pre block off @@ -1426,8 +1442,8 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: # TP=2 c64, TP=4 c128 and TP=2 c128. Every point is measured here rather # than combined from the earlier run: the image move leaves concurrency # 1-32 measured only on the superseded nightly-eed1f3d0 pin. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } # DeepSeek-V4.1-Flash on ATOM, following the upstream TP2 / TP4 AgentX recipe. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. @@ -1507,4 +1523,4 @@ dsv41flash-fp4-mi355x-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.60 search-space: - - { tp: 4, ep: 4, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 4, ep: 4, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml } diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index e38ba4bc9e..30ce89afcd 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -931,9 +931,9 @@ dsv4-fp4-b200-sglang-agentic-hicache-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 3, 4, 5] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [8, 10, 16] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [64, 96, 128, 160], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 3, 4, 5], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [8, 10, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [64, 96, 128, 160], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml } dsv4-fp4-b200-vllm-agentic-mtp: image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-591bb95 @@ -948,9 +948,9 @@ dsv4-fp4-b200-vllm-agentic-mtp: - dram-utilization: 0.90 search-space: # TP8 resident KV + DSpark6 - - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 6, 10, 14, 16] } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 6, 10, 14, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml } # DEP8 SimpleCPU + DSpark6 - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [32, 64, 96, 128, 160, 192], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [32, 64, 96, 128, 160, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml } # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 # does not have a B300-specific recipe, so this config reuses the existing DSR1 FP4 @@ -1130,8 +1130,8 @@ dsv4-fp4-b300-sglang-agentic-hicache-mtp: agentic-coding: - dram-utilization: 0.95 search-space: - - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [32, 64, 128, 256, 384, 512, 576], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [32, 64, 128, 256, 384, 512, 576], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml } # DeepSeek-V4-Pro-0813 on B300 with DSpark speculative decoding. Recipe is # selected inside benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh by @@ -1188,10 +1188,10 @@ qwen3.5-fp8-b200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64] } - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32] } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml } qwen3.5-fp4-b200-sglang: image: lmsysorg/sglang:v0.5.19-cu130 @@ -1529,7 +1529,7 @@ kimik3-fp4-b300-vllm-agentic-dspark: # One entry for every arm: the recipe drafts at DSpark 7 up to conc 8, # DSpark 3 at conc 16, and not at all above. Keep the concurrencies # disjoint across arms so exp-names stay unique. - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 70] } + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 70], srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml } dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 @@ -1660,15 +1660,15 @@ dsv4-fp4-b300-vllm-agentic-mtp: - dram-utilization: 0.95 search-space: # TP8 GPU-resident + MTP (num_speculative_tokens=3) - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4] } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml } # TP4 GPU-resident + MTP (num_speculative_tokens=3) - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml } # TP4 SimpleCPU + MTP (num_speculative_tokens=3) - - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [16] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml } # DEP4 SimpleCPU + MTP (num_speculative_tokens=3) - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [48, 64], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [48, 64], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml } # DEP8 SimpleCPU + MTP (num_speculative_tokens=3) - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [128, 256, 384, 512, 576], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [128, 256, 384, 512, 576], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml } qwen3.5-fp8-h200-sglang: image: lmsysorg/sglang:v0.5.14-cu130 @@ -4613,7 +4613,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_dep1_tep2_c44_b8" decode: num-worker: 1 tp: 2 @@ -4630,7 +4630,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p7d_dep4_tep8_c7_b1" decode: num-worker: 7 tp: 8 @@ -4647,7 +4647,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p2d_dep1_tep2_c52_b4" decode: num-worker: 2 tp: 2 @@ -4664,7 +4664,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 2 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p3d_tep2_tep8_c96_b128" decode: num-worker: 3 tp: 8 @@ -4681,7 +4681,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_3p1d_dep4_dep16_c565_b8" decode: num-worker: 1 tp: 16 @@ -4698,7 +4698,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_3p2d_dep4_dep4_c704_b32" decode: num-worker: 2 tp: 4 @@ -4729,7 +4729,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c1" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -4741,7 +4741,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c24" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -4753,7 +4753,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c32" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -4765,7 +4765,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c40" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -4777,7 +4777,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c48" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -4789,7 +4789,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c52" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -4801,7 +4801,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml:override_c64" # The seven disaggregated frontier points use six TP4 shapes plus one TP2 # shape. Stable X-Dynamo-Session-ID affinity replaces the removed conv-aware # routing message path. @@ -4830,7 +4830,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c8" decode: num-worker: 1 tp: 4 @@ -4846,7 +4846,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c16" decode: num-worker: 1 tp: 4 @@ -4862,7 +4862,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c32" decode: num-worker: 1 tp: 4 @@ -4878,7 +4878,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c64" decode: num-worker: 1 tp: 4 @@ -4894,7 +4894,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c96" decode: num-worker: 1 tp: 4 @@ -4910,7 +4910,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp4_tp4_c128" decode: num-worker: 1 tp: 4 @@ -4926,7 +4926,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_tp2_tp2_c72" decode: num-worker: 1 tp: 2 @@ -4963,7 +4963,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml:override_3p2d_c704" decode: num-worker: 2 tp: 4 @@ -4981,7 +4981,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml:override_3p4d_c565" decode: num-worker: 4 tp: 4 @@ -5014,7 +5014,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml:override_tp2_c44_b2" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -5027,7 +5027,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml:override_tp2_c44_b1" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -5040,7 +5040,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml:override_tp8_c7_b1" qwen3.5-fp8-b300-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 @@ -5054,9 +5054,9 @@ qwen3.5-fp8-b300-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 64, 68, 72] } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32] } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 34, 36, 38, 40, 44, 48, 52, 56] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 64, 68, 72], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 34, 36, 38, 40, 44, 48, 52, 56], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml } qwen3.5-fp4-b300-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 @@ -5070,9 +5070,9 @@ qwen3.5-fp4-b300-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 64, 68, 72] } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32] } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [36, 44, 52] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 64, 68, 72], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [36, 44, 52], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml } # Qwen3.8-Flash-Next NVFP4 AgentX on B300 via SGLang with native NEXTN MTP. # Day-zero recipe; mirrors the B200 arm. TP1 per the cookbook's verified @@ -5089,7 +5089,7 @@ qwen3.8next-fp4-b300-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } + - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml } # Controlled AgentX power A/B: identical software, topology, MTP settings, # concurrency, and memory tier across FP8 and FP4. The HBM-only rows measure # the natural AgentX prefix-cache workload; HiCache isolates host-tier effects. @@ -5153,7 +5153,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p6d_dep4_tp4" decode: num-worker: 6 tp: 4 @@ -5169,7 +5169,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p6d_dep4_tp4" decode: num-worker: 6 tp: 4 @@ -5186,7 +5186,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_4p1d_dep4_dep8_24_c4096" decode: num-worker: 1 tp: 8 @@ -5205,8 +5205,8 @@ qwen3.5-fp8-h100-sglang-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 14, 16] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [12, 14, 16, 20, 24, 28, 32, 42] } + - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 14, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml } + - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [12, 14, 16, 20, 24, 28, 32, 42], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml } # MTP speculative-decoding (spec-decoding: mtp) variant of # qwen3.5-fp8-h100-sglang-agentic: same TP8/EP8 GPU-resident and HiCache arms, @@ -5240,8 +5240,8 @@ qwen3.5-fp8-h200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 8, 12, 16] } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml } # Qwen3.8-Flash-Next FP8 AgentX on H200 via SGLang with native NEXTN MTP. # Day-zero recipe. H200 is Hopper, so this arm is FP8: NVFP4 is greyed out for @@ -5260,7 +5260,7 @@ qwen3.8next-fp8-h200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } + - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml } # H200 AgentX MTP frontier with DRAM HiCache. This is intentionally an MTP-only # submission; the model's non-speculative AgentX arm is not included. qwen3.5-fp8-h200-sglang-agentic-hicache-mtp: @@ -5275,7 +5275,7 @@ qwen3.5-fp8-h200-sglang-agentic-hicache-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [2, 4, 8, 10, 12, 16, 20, 24] } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [2, 4, 8, 10, 12, 16, 20, 24], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml } qwen3.5-fp8-h100-sglang-agentic-mtp: image: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 @@ -5289,8 +5289,8 @@ qwen3.5-fp8-h100-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml } # Qwen3.8-Flash-Next FP8 AgentX on H100 via SGLang with native NEXTN MTP. # Day-zero recipe. H100 is Hopper, so FP8: NVFP4 needs SM100 tensor cores. The @@ -5421,8 +5421,8 @@ minimaxm3-fp8-h100-vllm-agentic-mtp: # The fast sweep places the resident HBM cliff between c5 and c6. - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [6, 8] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [6, 8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml } minimaxm3-fp8-h200-vllm-agentic-mtp: image: vllm/vllm-openai:nightly-d9105ea8001e0a6d77a96327d17515bb5791fb36 @@ -5437,8 +5437,8 @@ minimaxm3-fp8-h200-vllm-agentic-mtp: # Keep the resident c1-c10 points and the Mooncake DRAM-offload c12 point. - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10] } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml } qwen3.5-fp4-b200-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 @@ -5452,10 +5452,10 @@ qwen3.5-fp4-b200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72] } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14] } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml } # Qwen3.8-Flash-Next NVFP4 AgentX on B200 via SGLang with native NEXTN MTP. @@ -5472,7 +5472,7 @@ qwen3.8next-fp4-b200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16] } + - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml } qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 @@ -5498,7 +5498,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp4" - spec-decoding: mtp kv-offloading: none conc-list: [2] @@ -5509,7 +5509,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp4_no_symm" - spec-decoding: mtp kv-offloading: none conc-list: [24] @@ -5520,7 +5520,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp4_parity" # Keep only the reproducible TP2/EP2 HiCache K5 transition point; C28 # was dominated by the published C28 result in the official PR sweep. - spec-decoding: mtp @@ -5534,7 +5534,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 2 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp2ep2_hicache" # Keep the published C32 interactivity anchor. The tighter capacity # layout becomes Pareto-relevant at C40 and remains deployable at C48. - spec-decoding: mtp @@ -5548,7 +5548,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 2 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp2ep2_hicache_k3_baseline" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -5560,7 +5560,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 2 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp2ep2_hicache_cap48" minimaxm3-fp4-b300-vllm-agentic-mtp: image: vllm/vllm-openai:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 model: nvidia/MiniMax-M3-NVFP4 @@ -5573,12 +5573,12 @@ minimaxm3-fp4-b300-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.683 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1] } - - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20] } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [30] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [30], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } - dram-utilization: 1.0 search-space: - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24] } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 @@ -5605,8 +5605,8 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 4 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml" - - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tep4_tp4_c1" + - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tep4_tp4_c1_eval" decode: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -5618,8 +5618,8 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml" - - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p3d_dep4_tp4_c24" + - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p3d_dep4_tp4_c24_eval" decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -5631,8 +5631,8 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml" - - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tp2_tp4_c20_c24" + - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tp2_tp4_c20_c24_eval" decode: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -5644,8 +5644,8 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml" - - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p3d_tp2_tp2_c48" + - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p3d_tp2_tp2_c48_eval" decode: { num-worker: 3, tp: 2, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -5657,8 +5657,8 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml" - - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p5d_tp2_tp2_c120" + - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p5d_tp2_tp2_c120_eval" decode: { num-worker: 5, tp: 2, ep: 1, dp-attn: false } minimaxm3-fp4-b300-trtllm-agentic-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 @@ -5672,9 +5672,9 @@ minimaxm3-fp4-b300-trtllm-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [20, 25, 30] } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [5, 10, 15, 20, 25, 30] } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5] } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [20, 25, 30], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [5, 10, 15, 20, 25, 30], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml } # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve # covers the middle/high-interactivity range omitted by the one-decode DEP @@ -5727,11 +5727,11 @@ minimaxm3-fp4-b200-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.683 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1] } - - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml } - dram-utilization: 1.0 search-space: - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [15, 20, 25, 30, 32, 34, 36, 38, 40] } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [15, 20, 25, 30, 32, 34, 36, 38, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml } minimaxm3-fp4-b200-trtllm-agentic-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 model: nvidia/MiniMax-M3-NVFP4 @@ -5747,8 +5747,8 @@ minimaxm3-fp4-b200-trtllm-agentic-mtp: # does not size the pool. - dram-utilization: 0.8 search-space: - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [5, 10, 15, 20, 25, 30, 35, 40, 45] } - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5] } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [5, 10, 15, 20, 25, 30, 35, 40, 45], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml } # Run the GB200 aggregate configuration with TP4 resident, TP8 resident across # two NVL4 nodes, and TP4 lazy SimpleCPU KV offload variants. The pinned # runtime uses Dynamo-native MiniMax parsing on the worker without the vLLM @@ -5777,7 +5777,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml:override_tp4" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: vllm-simple } @@ -5789,7 +5789,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml:override_tp4_vllm_simple" - spec-decoding: mtp kv-offloading: none conc-list: [1] @@ -5800,7 +5800,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml:override_tp8" minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: image: vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6 @@ -5828,7 +5828,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: ep: 4 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml:override_1p1d_tp8_c1" decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -5840,7 +5840,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: ep: 4 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml:override_1p2d_c8_c16" decode: { num-worker: 2, tp: 4, ep: 4, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -5852,7 +5852,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: ep: 4 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml:override_1p1d_c24" decode: { num-worker: 1, tp: 4, ep: 4, dp-attn: false } minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: @@ -5880,7 +5880,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c5_b5" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } @@ -5892,7 +5892,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c10_b10" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } @@ -5904,7 +5904,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c15_b15" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } @@ -5916,7 +5916,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c20_b20" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } @@ -5928,7 +5928,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c25_b25" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } @@ -5940,7 +5940,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c30_b30" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: native } @@ -5952,7 +5952,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml:override_c40_b40" dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 @@ -5978,7 +5978,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml:override_tp8" - spec-decoding: mtp conc-list: [8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80] num-nodes: 2 @@ -5988,7 +5988,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml:override_dep8" dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 model: deepseek-ai/DeepSeek-V4-Pro @@ -6012,7 +6012,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml:override_1p1d" decode: num-worker: 1 tp: 8 @@ -6027,7 +6027,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml:override_2p1d" decode: num-worker: 1 tp: 8 @@ -6058,7 +6058,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml:override_tp8" - spec-decoding: mtp conc-list: [4] num-nodes: 2 @@ -6068,7 +6068,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml:override_tp8" - spec-decoding: mtp conc-list: [1, 2, 4, 6, 8] num-nodes: 1 @@ -6078,7 +6078,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml:override_tp4" dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro @@ -6107,7 +6107,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_2p1d_dep8_dep12_c1152" decode: num-worker: 1 tp: 12 @@ -6124,7 +6124,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_2p1d_dep8_dep16_c1024" decode: num-worker: 1 tp: 16 @@ -6141,7 +6141,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_1p1d_dep4_dep8_c256" decode: num-worker: 1 tp: 8 @@ -6158,7 +6158,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_1p1d_dep8_dep16_c512" decode: num-worker: 1 tp: 16 @@ -6190,7 +6190,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_1p1d_dep4_dep16_c128" decode: num-worker: 1 tp: 16 @@ -6207,7 +6207,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml:override_1p1d_dep4_dep16_c256" decode: num-worker: 1 tp: 16 @@ -6243,7 +6243,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml:override_c4" - spec-decoding: mtp conc-list: [4] num-nodes: 2 @@ -6253,7 +6253,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml:override_c4" - spec-decoding: mtp conc-list: [8] num-nodes: 2 @@ -6263,7 +6263,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml:override_c8" dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro @@ -6292,7 +6292,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml:override_2p1d_dep12_c576" decode: num-worker: 1 tp: 12 @@ -6309,7 +6309,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml:override_2p1d_dep16_c512" decode: num-worker: 1 tp: 16 @@ -6326,7 +6326,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml:override_1p1d_c256" decode: num-worker: 1 tp: 8 @@ -6343,7 +6343,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml:override_1p1d_c128" decode: num-worker: 1 tp: 8 @@ -6385,7 +6385,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml:override_tp16_latency" # Balanced: multi_node_tep strategy, TEP16 across four GB200 nodes. # https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tep - spec-decoding: mtp @@ -6397,7 +6397,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: ep: 16 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml:override_tep16_balanced" # Throughput oriented: official multi_node_dep strategy, DEP16 across # four GB200 nodes (TP4 x DP4 = EP16, one local DP rank per node). # The recipe allows up to 3600s for full-context saturation warmup to drain. @@ -6411,7 +6411,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: ep: 16 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml:override_dep16" # High-concurrency DEP16 with vLLM Simple CPU KV offloading. c384 # exercises 384 of the 393 AgentX trajectories and remains below the # aggregate max-num-seqs capacity of 512 (128 per DP rank). @@ -6426,7 +6426,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: ep: 16 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml:override_dep16_vllm_simple_offload" # Kimi-K3 GB200 TP16/DCP16 profiles using Mooncake DRAM offload. kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-agg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef @@ -6506,7 +6506,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p4d_dep4_tep8_c4_b1" - "SLURM_PARTITION=batch_1" decode: num-worker: 4 @@ -6523,7 +6523,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p6d_dep4_tep4_c24_b4" - "SLURM_PARTITION=batch_1" decode: num-worker: 6 @@ -6540,7 +6540,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_dep8_dep32_c388_b4" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -6557,7 +6557,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_2p1d_dep8_dep32_c736_b8" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -6574,7 +6574,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_3p1d_dep8_dep16_c1152_b32" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -6591,7 +6591,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_5p1d_dep8_dep16_c2626_b96" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -6702,7 +6702,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml:override_tp8" - search-space: - spec-decoding: draft_model conc-list: [8] @@ -6713,7 +6713,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml:override_tp4" dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21 model: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -6739,7 +6739,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_c480" decode: num-worker: 1 tp: 16 @@ -6755,7 +6755,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_2p1d_c960" decode: num-worker: 1 tp: 16 @@ -6771,7 +6771,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_3p1d_c1440" decode: num-worker: 1 tp: 16 @@ -6787,7 +6787,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml:override_4p1d_c1920" decode: num-worker: 1 tp: 16 @@ -7095,7 +7095,7 @@ glm5.2-fp4-b300-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48], srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml } # First GLM-5.2 FP8 AgentX recipe on B300, the FP8 precision sibling of # glm5.2-fp4-b300-sglang-agentic-mtp and the B300 counterpart of @@ -7126,7 +7126,7 @@ glm5.2-fp8-b300-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml } # First GLM-5.2 AgentX recipe on B200, shipped spec-decode-only per the AgentX # policy that agentic arms enable speculative decoding rather than running a @@ -7155,7 +7155,7 @@ glm5.2-fp4-b200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml } # First GLM-5.2 FP8 AgentX recipe on B200, the FP8 precision sibling of # glm5.2-fp4-b200-sglang-agentic-mtp. Same spec-decode-only shape per the AgentX @@ -7185,7 +7185,7 @@ glm5.2-fp8-b200-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml } # GLM-5.2 NVFP4 B200 AgentX on Dynamo + SGLang. EAGLE uses the model's # built-in nextn head, with acceptance pinned to the golden thinking-on AL in @@ -7216,7 +7216,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml:override_c1" - spec-decoding: mtp conc-list: [4] kv-offloading: dram @@ -7228,7 +7228,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml:override_c4" - spec-decoding: mtp conc-list: [8] kv-offloading: dram @@ -7240,7 +7240,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml:override_c8" glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-20260910-00840301 @@ -7268,7 +7268,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml:override_1p4d_tp4_c48" decode: num-worker: 4 tp: 4 @@ -7285,7 +7285,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml:override_1p1d_c64" decode: num-worker: 1 tp: 8 @@ -7382,7 +7382,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 8 dp-attn: true additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml:override_1p6d_tp4_c45 decode: num-worker: 6 tp: 4 @@ -7400,7 +7400,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 8 dp-attn: true additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml:override_1p4d_tp4_c48 decode: num-worker: 4 tp: 4 @@ -7418,7 +7418,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 8 dp-attn: true additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml:override_2p1d_dep16_c128 decode: num-worker: 1 tp: 16 @@ -7456,7 +7456,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml:override_c2 - spec-decoding: mtp kv-offloading: dram kv-offload-backend: @@ -7470,7 +7470,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml:override_c4 - spec-decoding: mtp kv-offloading: dram kv-offload-backend: @@ -7484,7 +7484,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml:override_c8 glm5.2-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 model: nvidia/GLM-5.2-NVFP4 @@ -7639,7 +7639,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tp8_c1_b1_mtp5" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -7656,7 +7656,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p4d_tep4_c30_b2_mtp5" - "SLURM_PARTITION=batch_1" decode: num-worker: 4 @@ -7673,7 +7673,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_1p1d_tep8_c20_b5_mtp5" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -7690,7 +7690,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_3p4d_tep4_c60_b5_mtp5" - "SLURM_PARTITION=batch_1" decode: num-worker: 4 @@ -7707,7 +7707,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_5p1d_dep16_c260_b16_mtp3" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -7724,7 +7724,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml:override_4p1d_dep8_c227_b16_mtp3" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -8326,7 +8326,7 @@ dsv41flash-fp4-h100-vllm-agentic-dspark: # Extend beyond the c1 full-context capacity estimate to measure the # AgentX workload's practical saturation curve; c20, c24 and c28 fill in # the knee between c16 and the point where 1M-token trajectories preempt. - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 20, 24, 28] } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 20, 24, 28], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml } # SGLang arm for DeepSeek-V4.1-Flash AgentX on H100, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). @@ -8351,7 +8351,7 @@ dsv41flash-fp4-h100-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 20] } + - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml } # Sweep supported DP8 attention alongside the retained plain TP8 curve. dsv41flash-fp4-h100-sglang-agentic-dspark-dpa: @@ -8366,7 +8366,7 @@ dsv41flash-fp4-h100-sglang-agentic-dspark-dpa: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [4, 8, 16, 20] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [4, 8, 16, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml } dsv41flash-fp4-gb200-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 @@ -8381,13 +8381,13 @@ dsv41flash-fp4-gb200-vllm-agentic-dspark: - dram-utilization: 0.80 search-space: # Engram weights use UVA DRAM; the KV cache stays GPU-resident. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml } # TP2 halves the GPU count per replica. Weights rise to ~145 GiB on each # 256 GiB GPU with the Engram tables in pinned host DRAM, so the arm # takes the same caps as the B200 TP2 arm: batched tokens 4096 (the # indexer's logits buffer is 32 GiB at the upstream 16384) and graph # capture stopped at 512, leaving ~49 GiB of KV per GPU. - - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml } # SGLang arm for DeepSeek-V4.1-Flash AgentX on GB200, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). @@ -8411,8 +8411,8 @@ dsv41flash-fp4-gb200-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - - { tp: 2, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml } dsv41flash-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:deepseekv41-flash-0909 @@ -8428,9 +8428,9 @@ dsv41flash-fp4-b300-vllm-agentic-dspark: search-space: # Engram weights use UVA DRAM; the KV cache stays GPU-resident. The B300 # script uses explicit piecewise CUDA graph capture tiers. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml } # Run the same B300 recipe on two GPUs. - - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [2, 4, 8, 16, 32, 64, 128] } + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml } dsv41flash-fp4-b200-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 @@ -8445,8 +8445,8 @@ dsv41flash-fp4-b200-vllm-agentic-dspark: - dram-utilization: 0.80 search-space: # Engram weights use UVA DRAM; the KV cache stays GPU-resident. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml } + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml } # SGLang arm for DeepSeek-V4.1-Flash AgentX on B200, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). @@ -8469,8 +8469,8 @@ dsv41flash-fp4-b200-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8] } + - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml } dsv41flash-fp4-gb300-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-af1c01499b289be555c475669ba50a88e96d846e @@ -8485,13 +8485,13 @@ dsv41flash-fp4-gb300-vllm-agentic-dspark: - dram-utilization: 0.80 search-space: # Engram weights use UVA DRAM; the KV cache stays GPU-resident. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml } # TP2 halves the GPU count per replica. Weights rise to ~175 GiB on each # 277 GiB GPU with the Engram tables in pinned host DRAM, so the arm # takes the same caps as the B200 TP2 arm: batched tokens 4096 (the # indexer's logits buffer is 32 GiB at the upstream 16384) and graph # capture stopped at 512, leaving ~36 GiB of KV per GPU. - - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml } # H200 AgentX arm for DeepSeek-V4.1-Flash. Upstream marks h200 verified and says # the GB200 NVL4 TP4 layout becomes TP8 on 8-GPU nodes, so this is TP8. @@ -8515,13 +8515,13 @@ dsv41flash-fp4-h200-vllm-agentic-dspark: search-space: # 8x141 GB holds the 511 GB checkpoint minus the UVA-offloaded Engram # tables, so the KV cache stays GPU-resident across the full range. - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml } # TP4 doubles the per-GPU share to ~99 GiB of weights on each 141 GB H200 # with the Engram tables offloaded, leaving ~15 GiB of KV per GPU (~17M # tokens across the four ranks, about 3x the pool that served c1-c128 on # B200 TP2). The script caps batched tokens and the scheduler batch on # this arm so the sparse-attention indexer buffer does not consume it. - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml } # SGLang arm for DeepSeek-V4.1-Flash AgentX on H200, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). @@ -8544,8 +8544,8 @@ dsv41flash-fp4-h200-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml } + - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml } dsv4-fp4-b200-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 @@ -8574,7 +8574,7 @@ dsv4-fp4-b200-dynamo-sglang-agentic-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c1-mtp-hicache.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml:override_c1" - spec-decoding: mtp conc-list: [4] kv-offloading: dram @@ -8588,7 +8588,7 @@ dsv4-fp4-b200-dynamo-sglang-agentic-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c4-mtp-hicache.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml:override_c4" - spec-decoding: mtp conc-list: [8] kv-offloading: dram @@ -8602,7 +8602,7 @@ dsv4-fp4-b200-dynamo-sglang-agentic-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c8-mtp-hicache.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml:override_c8" dsv4-fp4-b200-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 @@ -8631,7 +8631,7 @@ dsv4-fp4-b200-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml:override_1p1d_c64" decode: num-worker: 1 tp: 8 @@ -8649,7 +8649,7 @@ dsv4-fp4-b200-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml:override_1p1d_c128" decode: num-worker: 1 tp: 8 @@ -8667,7 +8667,7 @@ dsv4-fp4-b200-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml:override_2p1d_c256" decode: num-worker: 1 tp: 8 @@ -8831,7 +8831,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp4_tp4_c4 decode: num-worker: 1 tp: 4 @@ -8849,7 +8849,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp4_tp4_c12 decode: num-worker: 1 tp: 4 @@ -8867,7 +8867,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp4_tp4_c24 decode: num-worker: 1 tp: 4 @@ -8885,7 +8885,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 4 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp4ep4_tp4_c32 decode: num-worker: 1 tp: 4 @@ -8903,7 +8903,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 2 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp2ep2_tp2ep2_c32 decode: num-worker: 1 tp: 2 @@ -8921,7 +8921,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 2 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp2ep2_tp2ep2_c40 decode: num-worker: 1 tp: 2 @@ -8939,7 +8939,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 2 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp2ep2_tp2ep2_c44 decode: num-worker: 1 tp: 2 @@ -8957,7 +8957,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 2 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp2ep2_tp2ep2_c48 decode: num-worker: 1 tp: 2 @@ -8975,7 +8975,7 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 2 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache.yaml + - CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml:override_tp2ep2_tp2ep2_c56_replayssm decode: num-worker: 1 tp: 2 @@ -8997,8 +8997,8 @@ dsv41flash-fp4-b300-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml } # Official SGLang nightly; Engram weights in host DRAM, native DSpark draft. dsv41flash-fp4-gb300-sglang-agentic-dspark: @@ -9013,5 +9013,5 @@ dsv41flash-fp4-gb300-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index 06c9ef6cf9..3fbd3adf07 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -191,7 +191,7 @@ Retain `meta_env.json`, `results*.json`, and `sample*.jsonl`. Agentic SWE-bench AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L2104-L2128)). -For multi-node srt-slurm jobs, the benchmark client may run on a different host from the frontend. `agentic_srt.sh` uses an explicit `AIPERF_SERVER_URL` when supplied, otherwise derives it from `SRT_FRONTEND_HOST` and `SRT_FRONTEND_PORT`, and falls back to `localhost:$PORT` only when no remote endpoint is available. Trace replay and inter-point drain checks must use that same resolved endpoint. +For multi-node srt-slurm jobs, the benchmark client may run on a different host from the frontend. `srt_agentic.sh` uses an explicit `AIPERF_SERVER_URL` when supplied, otherwise derives it from `SRT_FRONTEND_HOST` and `SRT_FRONTEND_PORT`, and falls back to `localhost:$PORT` only when no remote endpoint is available. Trace replay and inter-point drain checks must use that same resolved endpoint. Keep non-index engine or router wheels reproducible and immutable: check in the source patch and builder beside the launcher, verify the upstream wheel's digest before patching, assign an explicit local version, and install the published artifact through an exact URL with a SHA256 fragment. A local backport must not use an unreleased upstream version number. diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index 03c0580439..2105ba5d89 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -189,7 +189,7 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \ AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[fast replay 设置](../benchmarks/benchmark_lib.sh#L2104-L2128)）。 -对于多节点 srt-slurm 作业，benchmark client 与 frontend 可能运行在不同主机上。`agentic_srt.sh` 会优先使用显式提供的 `AIPERF_SERVER_URL`；否则从 `SRT_FRONTEND_HOST` 和 `SRT_FRONTEND_PORT` 推导地址；仅在没有远端 endpoint 时回退到 `localhost:$PORT`。Trace replay 和并发点之间的 drain 检查必须使用同一个解析后的 endpoint。 +对于多节点 srt-slurm 作业，benchmark client 与 frontend 可能运行在不同主机上。`srt_agentic.sh` 会优先使用显式提供的 `AIPERF_SERVER_URL`；否则从 `SRT_FRONTEND_HOST` 和 `SRT_FRONTEND_PORT` 推导地址；仅在没有远端 endpoint 时回退到 `localhost:$PORT`。Trace replay 和并发点之间的 drain 检查必须使用同一个解析后的 endpoint。 对于未发布到 package index 的 engine 或 router wheel，必须保证构建可复现且 artifact 不可变：在 launcher 旁签入源码 patch 与构建器，打 patch 前校验上游 wheel 的 digest，分配明确的 local version，并通过带 SHA256 fragment 的精确 URL 安装已发布 artifact。本地 backport 不得冒用尚未发布的上游版本号。 diff --git a/infx/matrix/generate.py b/infx/matrix/generate.py index af7ec486fd..28d0f2a1f3 100644 --- a/infx/matrix/generate.py +++ b/infx/matrix/generate.py @@ -223,6 +223,19 @@ def _worker_node_override(worker: dict, setting_name: str) -> int | None: return values[0] +def _merge_recipe(base: dict, override: dict) -> dict: + """Deep-merge an srt-slurm override variant over its base, as srtctl does.""" + merged = dict(base) + for key, value in override.items(): + if value is None: + merged.pop(key, None) + elif isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = _merge_recipe(merged[key], value) + else: + merged[key] = value + return merged + + def recipe_node_count(prefill: dict, decode: dict) -> int | None: """Read the authoritative node count from a checked-in srt-slurm recipe.""" config_files = { @@ -236,7 +249,7 @@ def recipe_node_count(prefill: dict, decode: dict) -> int | None: if len(config_files) != 1: raise ValueError(f"Conflicting CONFIG_FILE settings: {sorted(config_files)}") - config_file = config_files.pop() + config_file, _, selector = config_files.pop().partition(":") repo_root = repository_root() recipe_root = repo_root / "benchmarks" / "multi_node" / "srt-slurm-recipes" if config_file.startswith("benchmarks/multi_node/srt-slurm-recipes/"): @@ -249,12 +262,20 @@ def recipe_node_count(prefill: dict, decode: dict) -> int | None: return None recipe = yaml.safe_load(recipe_path.read_text()) + if "base" in recipe: + # srtctl merges a named variant over base (null deletes a key) and + # carries a top-level schema into it. Zip groups and non-schema-2 + # variant files have no authoritative count here; the selected master + # topology supplies the estimate. + if not (selector == "base" or (selector.startswith("override_") and selector in recipe)): + return None + schema = recipe.get("schema") + recipe = _merge_recipe(recipe["base"], recipe.get(selector) or {}) + recipe.setdefault("schema", schema) + if recipe.get("schema") != 2: + return None if recipe.get("schema") != 2: raise ValueError(f"srt-slurm recipes must declare schema: 2: {recipe_path}") - if "base" in recipe: - # A file with several override variants has no single authoritative - # node count. The selected master topology supplies the estimate. - return None roles = recipe.get("roles") if roles: # Schema 2 groups node allocations by role. A colocated decode role @@ -968,6 +989,8 @@ def _agentic_entries( Fields.CONC.value: conc, } ) + if benchmark.get(Fields.SRT_RECIPE.value) is not None: + entry[Fields.SRT_RECIPE.value] = benchmark[Fields.SRT_RECIPE.value] exp_name = ( f"{model_code}_tp{tp}_conc{conc}_" f"{agentic_kv_offload_suffix(kv_offloading, kv_offload_backend)}" diff --git a/infx/matrix/validation.py b/infx/matrix/validation.py index 7f774570b9..bf1af5f272 100644 --- a/infx/matrix/validation.py +++ b/infx/matrix/validation.py @@ -309,6 +309,7 @@ class SingleNodeAgenticMatrixEntry(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) image: str + srt_recipe: str | None = Field(default=None, alias=Fields.SRT_RECIPE.value, min_length=1) model: str model_prefix: str = Field(alias=Fields.MODEL_PREFIX.value) precision: str @@ -628,6 +629,7 @@ class AgenticCodingSearchSpaceEntry(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) tp: int | None = None + srt_recipe: str | None = Field(default=None, alias=Fields.SRT_RECIPE.value, min_length=1) pp: int = Field(default=1, gt=0, strict=True) dcp_size: int = Field(default=1, alias=Fields.DCP_SIZE.value, gt=0, strict=True) pcp_size: int = Field(default=1, alias=Fields.PCP_SIZE.value, gt=0, strict=True) diff --git a/infx/srt_slurm/single_node.py b/infx/srt_slurm/single_node.py index 31655f6dc7..1e6baece4b 100644 --- a/infx/srt_slurm/single_node.py +++ b/infx/srt_slurm/single_node.py @@ -1,4 +1,4 @@ -"""Bind a native single-node SRT recipe to one fixed-sequence matrix point.""" +"""Bind a native single-node SRT recipe to one fixed-sequence or AgentX matrix point.""" from __future__ import annotations @@ -37,6 +37,14 @@ def parallelism_constraints( "pipeline_parallel_size": (args.get("pipeline_parallel_size", 1), 1), "DP_ATTENTION": (args.get("enable_attention_dp", False), dp_attention), } + if engine == "vllm": + # vLLM spreads DP attention across data-parallel ranks of tensor size 1. + data_parallel = args.get("data-parallel-size", 1) + return { + "tensor x data parallel": (args.get("tensor-parallel-size", 1) * data_parallel, tp), + "DP_ATTENTION": (data_parallel > 1, dp_attention), + "enable-expert-parallel": (args.get("enable-expert-parallel", False), ep > 1), + } if engine == "atom": if ep not in {1, tp}: raise ValueError("ATOM expert parallelism must be 1 or TP") @@ -77,12 +85,16 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N workload = benchmark["env"] engine_config = recipe["engine"] engine = engine_config["type"] if isinstance(engine_config, dict) else engine_config - if environment["FRAMEWORK"] not in {"sglang", "trt", "atom"}: + if environment["FRAMEWORK"] not in {"sglang", "trt", "atom", "vllm"}: raise ValueError(f"Unsupported single-node framework: {environment['FRAMEWORK']!r}") spec = spec_parameters(role, engine) - if spec and spec["method"] not in {"eagle", "nextn", "mtp"}: - raise ValueError("Single-node SRT supports only native MTP or no speculation") - speculation = "mtp" if spec else "none" + if spec and spec["method"] not in {"eagle", "eagle3", "nextn", "mtp", "dspark"}: + raise ValueError( + "Single-node SRT supports only native MTP, EAGLE3, DSpark or no speculation" + ) + # A point that stops drafting may keep its matrix label. + speculation = "mtp" if spec else workload.get("SPEC_DECODING", "none") + agentic = environment["IS_AGENTIC"] == "1" expected = { "engine": (engine, SINGLE_NODE_ENGINES[environment["FRAMEWORK"]]), "model": (recipe["model"]["path"], f"hf:{environment['MODEL']}"), @@ -95,23 +107,29 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N "roles": (set(recipe["roles"]), {"agg"}), "benchmark type": (benchmark["type"], "custom"), "benchmark MODEL": (workload["MODEL"], environment["MODEL"]), - "SPEC_DECODING": (speculation, environment["SPEC_DECODING"]), - "USE_CHAT_TEMPLATE": (workload["USE_CHAT_TEMPLATE"], "true" if spec else "false"), + # draft_model names a bundled or separate draft; its recipes speculate natively. + "SPEC_DECODING": ( + speculation, + "mtp" + if environment["SPEC_DECODING"] == "draft_model" + else environment["SPEC_DECODING"], + ), + "AgentX client": (benchmark.get("command", "").endswith("srt_agentic.sh"), agentic), } - if "CONC" in workload: - expected["CONC"] = (str(workload["CONC"]), environment["CONC"]) + if not agentic: + expected["USE_CHAT_TEMPLATE"] = (workload["USE_CHAT_TEMPLATE"], "true" if spec else "false") + for name in ("ISL", "OSL", "RANDOM_RANGE_RATIO"): + expected[name] = (str(workload[name]), environment[name]) + # A variant that names its point, or the host budget it sizes, must match the matrix. + for name in ("CONC", "KV_OFFLOADING", "TOTAL_CPU_DRAM_GB"): + if name in workload: + expected[name] = (str(workload[name]), environment[name]) if engine == "atom": # Native ATOM derives -tp from the aggregate worker's GPU allocation. expected["ATOM TP"] = (role["gpus"], int(environment["TP"])) - for name in ("ISL", "OSL", "RANDOM_RANGE_RATIO"): - expected[name] = (str(workload[name]), environment[name]) - # Multi-node and AgentX workloads use their existing connector. - for name, value in { - "PP_SIZE": "1", - "DCP_SIZE": "1", - "PCP_SIZE": "1", - "IS_AGENTIC": "0", - }.items(): + # vLLM shards decode KV across its tensor-parallel ranks. + dcp = str(args.get("decode-context-parallel-size", 1)) if engine == "vllm" else "1" + for name, value in {"PP_SIZE": "1", "DCP_SIZE": dcp, "PCP_SIZE": "1"}.items(): expected[name] = (environment[name], value) for name, (actual, wanted) in expected.items(): if actual != wanted: @@ -141,14 +159,18 @@ def runtime_arguments(config: str, environment: Mapping[str, str]) -> list[str]: # flags. Runtime option mappings therefore need individual leaf sets. for key, value in options.items(): overrides += ["--set", f"srun_options.{key}={json.dumps(value)}"] - for name in ( + agentic = environment["IS_AGENTIC"] == "1" + names = [ "CONC", "RESULT_FILENAME", "GPU_MONITOR_INTERVAL", "RUN_EVAL", "EVAL_ONLY", "FRAMEWORK", - ): + ] + if agentic: + names += ["MODEL_PREFIX", "PRECISION", "DURATION", "TP", "PP_SIZE", "PCP_SIZE"] + for name in names: value = environment[name] if not value: raise ValueError(f"Missing runtime input: {name}") @@ -157,6 +179,10 @@ def runtime_arguments(config: str, environment: Mapping[str, str]) -> list[str]: if name == "CONC" and name in recipe["benchmark"]["env"]: continue overrides += ["--set", f"benchmark.env.{name}={json.dumps(value)}"] + if agentic: + # The aggregated result lands where fixed-sequence results do. + overrides += ["--set", 'benchmark.env.AGENTIC_OUTPUT_DIR="/logs"'] + return [*overrides, "--set", 'benchmark.env.RESULT_DIR="/logs/agentic"'] if environment["EVAL_ONLY"] == "true": context = int(environment["MAX_MODEL_LEN"]) if context <= 0: @@ -165,6 +191,7 @@ def runtime_arguments(config: str, environment: Mapping[str, str]) -> list[str]: "sglang": ("context-length",), "trt": ("max_seq_len", "max_num_tokens"), "atom": ("max-model-len",), + "vllm": ("max-model-len",), }[environment["FRAMEWORK"]] for key in context_keys: overrides += ["--set", f"roles.agg.args.{key}={context}"] diff --git a/infx/srt_slurm/synthetic_acceptance.py b/infx/srt_slurm/synthetic_acceptance.py index be7232eb12..724eb535d6 100644 --- a/infx/srt_slurm/synthetic_acceptance.py +++ b/infx/srt_slurm/synthetic_acceptance.py @@ -27,6 +27,7 @@ "dynamo-sglang": "sglang", "trt": "trtllm", "dynamo-trt": "trtllm", + "atom": "atom", } SGLANG_VARIABLES = ( "SGLANG_SIMULATE_ACC_LEN", @@ -42,10 +43,15 @@ def spec_parameters(role: Mapping[str, Any], engine: str) -> dict[str, Any]: method = args.get("method") if not method: return {} - return { + spec = { "method": str(method).lower(), "num_speculative_tokens": args.get("num-speculative-tokens"), + "model": args.get("draft-model", ""), } + if spec["method"] == "dspark": + # ATOM DSpark verifies with probabilistic block rejection sampling. + spec["draft_sample_method"] = "probabilistic" + return spec if engine == "vllm": raw = args.get("speculative-config") if raw is None: @@ -181,6 +187,13 @@ def build_overrides( "--set", f"{prefix}.args.speculative-config={json.dumps(worker_spec)}", ] + elif engine == "atom": + # ATOM forces acceptance with a server flag rather than environment. + key = "spec-decode-acceptance-length" + if al is not None and worker_spec: + overrides += ["--set", f"{prefix}.args.{key}={al:g}"] + elif key in (role.get("args") or {}): + overrides += ["--unset", f"{prefix}.args.{key}"] elif al is not None and worker_spec: values = ( (f"{al:g}", "match-expected", "real-draft-token") diff --git a/runners/launch_b200-cw.sh b/runners/launch_b200-cw.sh index 655599e755..60d86d26ff 100644 --- a/runners/launch_b200-cw.sh +++ b/runners/launch_b200-cw.sh @@ -6,7 +6,7 @@ check_env_vars IS_MULTINODE IS_AGENTIC EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi diff --git a/runners/launch_b200-nb.sh b/runners/launch_b200-nb.sh index 419af005de..cece9b5801 100644 --- a/runners/launch_b200-nb.sh +++ b/runners/launch_b200-nb.sh @@ -6,7 +6,7 @@ check_env_vars IS_MULTINODE IS_AGENTIC EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index a041b9f403..1c6c7afc04 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -50,7 +50,7 @@ if uses_native_srt_lane; then LAUNCH_PATH="native-srt" elif [[ "$IS_MULTINODE" == "true" ]]; then LAUNCH_PATH="multinode-srt" -elif [[ "$IS_AGENTIC" == "0" ]]; then +elif [[ "$IS_AGENTIC" == "0" || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE LAUNCH_PATH="native-single-node" else @@ -138,10 +138,8 @@ elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/scratch/models/Kimi-K3" export SRT_SLURM_MODEL_PREFIX="kimik3" elif [[ $MODEL_PREFIX == "qwen3.8next" && $PRECISION == "fp4" ]]; then - check_env_vars MODEL_PATH - if [[ -n "${MODEL_PATH}" && -d "$MODEL_PATH" ]]; then - : - else + # No pool setting names this checkpoint; default to the node-local copy. + if [[ -z "${MODEL_PATH:-}" || ! -d "$MODEL_PATH" ]]; then export MODEL_PATH="/scratch/models/Qwen3.8-Flash-Next-NVFP4" fi export SRT_SLURM_MODEL_PREFIX="qwen3.8next-fp4" @@ -155,6 +153,8 @@ fi if [[ "$LAUNCH_PATH" == native-single-node ]]; then HF_HUB_CACHE_MOUNT=/data/home/sa-shared/gharunners/hf-hub-cache SRT_MODEL_PATH="$MODEL_PATH" + # Models not staged locally resolve through the Hugging Face cache mount. + [[ "$SRT_MODEL_PATH" == /* ]] || SRT_MODEL_PATH="hf:$MODEL" SRT_SQUASH_FILE="$B200_SQUASH_DIR/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" launch_srt_single_node b200-nscale-slurm \ --var SLURM_ACCOUNT "$SLURM_ACCOUNT" --var SLURM_PARTITION "$SLURM_PARTITION" diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index c5f33162c4..00dbbfa6fe 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -127,7 +127,7 @@ if [[ "$IS_MULTINODE" == true ]]; then elif [[ -n "${BENCH_SCRIPT_OVERRIDE:-}" ]]; then # SPEED-Bench collectors explicitly supply their script outside this migration. EXECUTION_PATH=script -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi @@ -138,6 +138,13 @@ if [[ "$EXECUTION_PATH" == native-single-node ]]; then SRT_MODEL_PATH="$MODEL_ROOT/${MODEL##*/}" if [[ "$MODEL" == nvidia/DeepSeek-R1-0528-FP4-V2 ]]; then SRT_MODEL_PATH="$MODEL_ROOT/DeepSeek-R1-0528-NVFP4-v2" + elif [[ " ${STAGED_MODELS[*]} " != *" ${MODEL##*/} "* || "${MODEL##*/}" == DeepSeek-V4-Pro-0813 ]]; then + # Not staged on every node's NVMe; read the shared copy. + SRT_MODEL_PATH="$SHARED_MODEL_ROOT/${MODEL##*/}" + fi + # Not staged on node-local NVMe: the engine downloads it into the shared HF cache. + if [[ "$MODEL" == RadixArk/Qwen3.8-Flash-Next-NVFP4 ]]; then + SRT_MODEL_PATH="hf:$MODEL" fi SRT_SQUASH_FILE="$SQUASH_DIR/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" launch_srt_single_node b300-dsxe \ @@ -212,7 +219,7 @@ export OSL="$OSL" SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" echo "Creating srtslurm.yaml configuration..." write_srt_cluster_config b300-dsxe srtslurm.yaml "$USES_DCGM_POWER" \ - --var MODEL_ROOT "$MODEL_ROOT" || exit 1 + --var MODEL_ROOT "$MODEL_ROOT" --var SRT_DEFAULT_TIME_LIMIT "$SALLOC_TIME_LIMIT" || exit 1 echo "Generated srtslurm.yaml:" cat srtslurm.yaml diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index f26da84aff..66a0996ba3 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -95,6 +95,17 @@ import_squash() { ) || exit 1 } +# Single-tray points with an srt-slurm recipe run natively on the aarch64 trays. +if [[ "$IS_MULTINODE" != true && -n "${SRT_RECIPE:-}" ]]; then + HF_HUB_CACHE_MOUNT="/mnt/lustre01/users-public/sa-shared/hf-hub-cache" + SRT_MODEL_PATH="hf:$MODEL" + SRT_SQUASH_FILE="$SQUASH_DIR/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + import_squash "$SRT_SQUASH_FILE" "$IMAGE" + SRT_SETUP_ARCH=aarch64 launch_srt_single_node gb200-nv \ + --var SLURM_ACCOUNT "$SLURM_ACCOUNT" --var SLURM_PARTITION "$SLURM_PARTITION" + exit $? +fi + # Direct single-tray AgentX uses the existing shared image and HF caches. if [[ "$MODEL_PREFIX" == "dsv41flash" && ( "$FRAMEWORK" == "vllm" || "$FRAMEWORK" == "sglang" ) && "${IS_MULTINODE}" != "true" ]]; then check_env_vars SPEC_DECODING diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 2e5d4069dc..7d579742b0 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -105,6 +105,18 @@ import_squash() { } import_squash "$SQUASH_FILE" "$IMAGE" +# Single-tray points with an srt-slurm recipe run natively on the aarch64 trays. +if [[ "$IS_MULTINODE" != true && -n "${SRT_RECIPE:-}" ]]; then + HF_HUB_CACHE_MOUNT="$HF_HUB_CACHE_HOST_PATH" + SRT_MODEL_PATH="hf:$MODEL" + SRT_SQUASH_FILE="$SQUASH_FILE" + SRT_SETUP_ARCH=aarch64 launch_srt_single_node gb300-nv \ + --var SLURM_ACCOUNT "$SLURM_ACCOUNT" --var SLURM_PARTITION "$SLURM_PARTITION" \ + --var AIPERF_MMAP_CACHE_HOST_PATH "$AIPERF_MMAP_CACHE_HOST_PATH" \ + --var HF_HUB_CACHE_HOST_PATH "$HF_HUB_CACHE_HOST_PATH" \ + --var DYNAMO_WHEELS_CACHE_HOST_PATH "$DYNAMO_WHEELS_CACHE_HOST_PATH" + exit $? +fi # Keep this branch before the nginx import and srtctl setup. if [[ "$MODEL_PREFIX" == "dsv41flash" && ( "$FRAMEWORK" == "vllm" || "$FRAMEWORK" == "sglang" ) && "${IS_MULTINODE}" != "true" ]]; then check_env_vars SPEC_DECODING diff --git a/runners/launch_h100-cw.sh b/runners/launch_h100-cw.sh index 693211a505..a5cb2ae7f3 100644 --- a/runners/launch_h100-cw.sh +++ b/runners/launch_h100-cw.sh @@ -6,7 +6,7 @@ check_env_vars IS_MULTINODE IS_AGENTIC EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 18a153cccf..2cf9b193cc 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -17,7 +17,7 @@ set -x EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi @@ -26,6 +26,10 @@ if [[ "$EXECUTION_PATH" == native-single-node ]]; then HF_HUB_CACHE_MOUNT=/mnt/nfs/sa-shared/gharunners/hf-hub-cache SRT_MODEL_PATH="hf:$MODEL" SRT_SQUASH_FILE="/mnt/nfs/lustre/containers/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + # The job's host-side orchestrator cannot write the compute node's home. + # One cache per runner: concurrent builds in a shared NFS cache race. + export UV_CACHE_DIR="/mnt/nfs/sa-shared/.uv/cache-${RUNNER_NAME:?}" + export UV_PYTHON_INSTALL_DIR="/mnt/nfs/sa-shared/.uv/python" launch_srt_single_node h100-dgxc-slurm \ --var SLURM_ACCOUNT "$SLURM_ACCOUNT" --var SLURM_PARTITION "$SLURM_PARTITION" \ --var CONTAINER_KEY "$IMAGE" diff --git a/runners/launch_h200-cw.sh b/runners/launch_h200-cw.sh index d101457309..4ebe6b68bc 100644 --- a/runners/launch_h200-cw.sh +++ b/runners/launch_h200-cw.sh @@ -6,7 +6,7 @@ check_env_vars IS_MULTINODE IS_AGENTIC EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 55651913ba..5965c9176c 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -18,12 +18,13 @@ source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" || exit 1 EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi if [[ "$EXECUTION_PATH" == native-single-node ]]; then + SRT_MODEL_PATH="hf:$MODEL" SRT_SQUASH_FILE="/data/containers/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" launch_srt_single_node h200-dgxc-slurm \ --var SLURM_ACCOUNT "$SLURM_ACCOUNT" --var SLURM_PARTITION "$SLURM_PARTITION" \ diff --git a/runners/launch_mi300x-amd.sh b/runners/launch_mi300x-amd.sh index e881f41dc2..f413318c5b 100644 --- a/runners/launch_mi300x-amd.sh +++ b/runners/launch_mi300x-amd.sh @@ -8,7 +8,7 @@ set -eo pipefail EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi diff --git a/runners/launch_mi325x-amds.sh b/runners/launch_mi325x-amds.sh index 8063915ea5..1b6f29c993 100644 --- a/runners/launch_mi325x-amds.sh +++ b/runners/launch_mi325x-amds.sh @@ -8,7 +8,7 @@ set -eo pipefail EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index c97c7beb55..29282f9cc8 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -7,7 +7,7 @@ check_env_vars EVAL_ONLY IS_AGENTIC IS_MULTINODE KEEP_LOGS RUN_EVAL EXECUTION_PATH=agentic if [[ "$IS_MULTINODE" == true ]]; then EXECUTION_PATH=multinode -elif [[ "$IS_AGENTIC" == 0 ]]; then +elif [[ "$IS_AGENTIC" == 0 || -n "${SRT_RECIPE:-}" ]]; then check_env_vars SRT_RECIPE EXECUTION_PATH=native-single-node fi @@ -15,10 +15,22 @@ if [[ "$EXECUTION_PATH" == native-single-node ]]; then check_env_vars GITHUB_WORKSPACE MODEL IMAGE source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" || exit 1 export HF_HUB_CACHE_MOUNT=/var/lib/hf-hub-cache/ + # AgentX checkpoints the legacy scripts read from the shared NFS cache. + if [[ "$IS_AGENTIC" == 1 ]]; then + case "$MODEL" in + MiniMaxAI/MiniMax-M3*|amd/MiniMax-M3*|zai-org/GLM-5.2-FP8|deepseek-ai/DeepSeek-V4.1-Flash) + export HF_HUB_CACHE_MOUNT=/it-share/hf-hub-cache/ ;; + deepseek-ai/DeepSeek-V4-Pro|deepseek-ai/DeepSeek-V4-Pro-0813) + if [[ "$FRAMEWORK" == vllm || "$FRAMEWORK" == atom ]]; then + export HF_HUB_CACHE_MOUNT=/it-share/hf-hub-cache/ + fi ;; + esac + fi export SRT_MODEL_PATH="hf:$MODEL" export SALLOC_TIME_LIMIT=500 export SRT_SRUN_OPTIONS='{"container-remap-root":"", "container-writable":""}' - SRT_SQUASH_FILE="/var/lib/squash/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + # A squash staged on shared storage survives the registry pruning nightly tags. + SRT_SQUASH_FILE="/it-share/gharunners2/srt-slurm/containers/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" launch_srt_single_node mi355x-amds --var GITHUB_WORKSPACE "$GITHUB_WORKSPACE" exit $? fi diff --git a/runners/patch_vllm_simple_kv_offload.py b/runners/patch_vllm_simple_kv_offload.py deleted file mode 100755 index 872bd9960a..0000000000 --- a/runners/patch_vllm_simple_kv_offload.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python3 -"""Split heterogeneous vLLM KV backing storage into valid CPU offload regions.""" - -from __future__ import annotations - -import importlib.util -import sys -from pathlib import Path - -OLD_SETUP = """ logical_storage_bytes = self.kv_cache_config.kv_cache_tensors[0].size - - # The DMA backend copies whole blocks as base + block_id * stride(0), -""" -NEW_SETUP = """ logical_storage_bytes = self.kv_cache_config.kv_cache_tensors[0].size - split_storage_by_layer = any( - logical_storage_bytes - % (num_blocks * cache_tensor.block_stride) - != 0 - for cache_tensor in self.kv_cache_config.kv_cache_tensors - ) - - # The DMA backend copies whole blocks as base + block_id * stride(0), -""" -OLD_LOOP = """ unique_gpu_caches: dict[str, torch.Tensor] = {} - seen: set[tuple[torch.device, int]] = set() - for name, tensor in kv_caches.items(): - storage = tensor.untyped_storage() - key = (tensor.device, storage.data_ptr()) - if key in seen: - continue - seen.add(key) - - physical_per_block, remainder = divmod(tensor.shape[0], num_blocks) - assert remainder == 0, ( - f"KV cache {name!r} has {tensor.shape[0]} physical blocks, which " - f"is not divisible by {num_blocks} scheduler blocks" - ) - block_bytes = tensor.stride(0) * tensor.element_size() * physical_per_block - raw = torch.empty(0, dtype=torch.int8, device=tensor.device).set_(storage) - assert raw.numel() >= logical_storage_bytes, ( - f"KV cache {name!r} storage has {raw.numel()} bytes, smaller " - f"than the configured {logical_storage_bytes}-byte allocation" - ) - regions = raw[:logical_storage_bytes].view(-1, num_blocks, block_bytes) - for idx, region in enumerate(regions): - key_name = name if len(regions) == 1 else f"{name}.{idx}" - unique_gpu_caches[key_name] = region -""" -NEW_LOOP = """ unique_gpu_caches: dict[str, torch.Tensor] = {} - seen: set[tuple[torch.device, int, int, int]] = set() - for name, tensor in kv_caches.items(): - physical_per_block, remainder = divmod(tensor.shape[0], num_blocks) - assert remainder == 0, ( - f"KV cache {name!r} has {tensor.shape[0]} physical blocks, which " - f"is not divisible by {num_blocks} scheduler blocks" - ) - block_bytes = tensor.stride(0) * tensor.element_size() * physical_per_block - storage = tensor.untyped_storage() - raw = torch.empty(0, dtype=torch.int8, device=tensor.device).set_(storage) - - if split_storage_by_layer: - region_offset = tensor.storage_offset() * tensor.element_size() - region_bytes = num_blocks * block_bytes - else: - region_offset = 0 - region_bytes = logical_storage_bytes - - key = (tensor.device, storage.data_ptr(), region_offset, region_bytes) - if key in seen: - continue - seen.add(key) - - region_end = region_offset + region_bytes - assert raw.numel() >= region_end, ( - f"KV cache {name!r} storage has {raw.numel()} bytes, smaller " - f"than the required {region_end}-byte region" - ) - regions = raw[region_offset:region_end].view( - -1, num_blocks, block_bytes - ) - for idx, region in enumerate(regions): - key_name = name if len(regions) == 1 else f"{name}.{idx}" - unique_gpu_caches[key_name] = region -""" - - -def installed_worker_path() -> Path: - """Return the SimpleCPUOffload worker module from the installed vLLM.""" - spec = importlib.util.find_spec("vllm") - if spec is None or not spec.submodule_search_locations: - raise RuntimeError("vllm package is not installed") - package_root = Path(next(iter(spec.submodule_search_locations))) - return package_root / "v1/simple_kv_offload/worker.py" - - -def patch_worker(worker_path: Path) -> bool: - """Patch heterogeneous layer-region sizing and return whether source changed.""" - source = worker_path.read_text() - if NEW_SETUP in source and NEW_LOOP in source: - return False - if NEW_SETUP in source or NEW_LOOP in source: - raise RuntimeError(f"partially patched vLLM worker at {worker_path}") - if source.count(OLD_SETUP) != 1 or source.count(OLD_LOOP) != 1: - raise RuntimeError( - f"unsupported vLLM SimpleCPUOffload worker at {worker_path}" - ) - - patched = source.replace(OLD_SETUP, NEW_SETUP).replace(OLD_LOOP, NEW_LOOP) - worker_path.write_text(patched) - return True - - -def main(argv: list[str]) -> int: - if len(argv) > 2: - print(f"Usage: {argv[0]} [WORKER_PATH]", file=sys.stderr) - return 2 - - try: - worker_path = ( - Path(argv[1]).resolve() if len(argv) == 2 else installed_worker_path() - ) - changed = patch_worker(worker_path) - except (OSError, RuntimeError) as error: - print(f"ERROR: failed to patch vLLM CPU offload: {error}", file=sys.stderr) - return 1 - - state = "Patched" if changed else "Already patched" - print(f"{state} vLLM SimpleCPUOffload heterogeneous layer regions") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv)) diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 68aedf6a25..50370edfeb 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -150,7 +150,7 @@ apply_srt_recipe() { "$config" "$framework" -- "$@" } -# One native submission per fixed-sequence matrix point, shared across Slurm pools. +# One native submission per fixed-sequence or AgentX matrix point, shared across Slurm pools. launch_srt_single_node() { set -eo pipefail local profile="$1" @@ -193,7 +193,7 @@ launch_srt_single_node() { --var SRT_DEFAULT_TIME_LIMIT "$SALLOC_TIME_LIMIT" \ --model "hf:$MODEL" "$SRT_MODEL_PATH" --container "$IMAGE" "$SRT_CONTAINER" \ --mount "$HF_HUB_CACHE_MOUNT" "$HF_HUB_CACHE" --exclusive "$@" - run_srt_setup ARCH=x86_64 + run_srt_setup "ARCH=${SRT_SETUP_ARCH:-x86_64}" SRT_JOB_ID="" SRT_JOB_OUTPUT="" @@ -216,6 +216,10 @@ launch_srt_single_node() { [[ -f "$artifact" ]] || continue copy_to_workspace "$artifact" "$GITHUB_WORKSPACE/$(basename "$artifact")" || rc=1 done + # AgentX uploads its raw replay artifacts and power window from results/. + if [[ -d "$SRT_JOB_OUTPUT/logs/agentic" ]]; then + cp -r "$SRT_JOB_OUTPUT/logs/agentic" "$GITHUB_WORKSPACE/results" || rc=1 + fi fi exit "$rc" } diff --git a/runners/srt-slurm/b300-dsxe.yaml b/runners/srt-slurm/b300-dsxe.yaml index 9da5a6c698..a9ffae66cf 100644 --- a/runners/srt-slurm/b300-dsxe.yaml +++ b/runners/srt-slurm/b300-dsxe.yaml @@ -1,5 +1,7 @@ default_account: ${SLURM_ACCOUNT} default_partition: ${SLURM_PARTITION} +# srtctl otherwise submits with a one-hour limit. +default_time_limit: ${SRT_DEFAULT_TIME_LIMIT} gpus_per_node: 8 network_interface: "" srtctl_root: ${SRTCTL_ROOT} diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index c52883d20e..84ae7d0567 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -20,7 +20,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] BENCHMARK_LIB = REPO_ROOT / "benchmarks" / "benchmark_lib.sh" -MULTINODE_AGENTIC_SCRIPT = REPO_ROOT / "benchmarks/multi_node/agentic_srt.sh" +MULTINODE_AGENTIC_SCRIPT = REPO_ROOT / "benchmarks/srt_agentic.sh" @pytest.fixture(autouse=True) diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index a09db34998..732827c59b 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -121,6 +121,30 @@ def test_multinode_node_count_reads_schema_two_roles(tmp_path, monkeypatch, role assert generate.recipe_node_count(prefill, {}) == expected +@pytest.mark.parametrize("selector, expected", [ + ("base", 3), + ("override_wide", 5), + ("override_colocated", 2), + ("zip_override_sweep[0]", None), +]) +def test_recipe_node_count_resolves_override_selectors(tmp_path, monkeypatch, selector, expected): + recipe = tmp_path / "benchmarks/multi_node/srt-slurm-recipes/variants.yaml" + recipe.parent.mkdir(parents=True) + recipe.write_text(yaml.safe_dump({ + "schema": 2, + "base": {"roles": {"prefill": {"nodes": 1}, "decode": {"nodes": 2}}}, + "override_wide": {"roles": {"decode": {"nodes": 4}}}, + "override_colocated": {"roles": {"prefill": {"nodes": 2}, "decode": {"nodes": "colocate"}}}, + "zip_override_sweep": {"roles": {"decode": {"nodes": [1, 2]}}}, + })) + import infx.matrix.generate as generate + import infx.config + (tmp_path / "configs").mkdir() + monkeypatch.setattr(infx.config, "__file__", str(tmp_path / "infx/config.py")) + prefill = {"additional-settings": [f"CONFIG_FILE=recipes/variants.yaml:{selector}"]} + assert generate.recipe_node_count(prefill, {}) == expected + + def test_multinode_node_count_uses_role_gpu_footprints(sample_runner_config): prefill = {"num-worker": 3, "tp": 2, "pp": 1, "pcp-size": 1} decode = {"num-worker": 2, "tp": 8, "pp": 1, "pcp-size": 1} diff --git a/utils/test_srt_single_node.py b/utils/test_srt_single_node.py index b1c31b9a29..2eb047749a 100644 --- a/utils/test_srt_single_node.py +++ b/utils/test_srt_single_node.py @@ -335,7 +335,7 @@ def test_pool_launcher_stages_artifacts_and_propagates_failure(point, tmp_path, env.pop("SRT_RECIPE") if failure == "agentic": env.update(IS_AGENTIC="1", SCENARIO_SUBDIR="agentic/", EXP_NAME="fixture_agentic", - RUNNER_NAME="fixture_00", SRT_RECIPE="unused.yaml") + RUNNER_NAME="fixture_00", SRT_RECIPE="") # not yet ported: legacy script result = subprocess.run( ["bash", str(ROOT / f"runners/launch_{pool}.sh")], cwd=tmp_path, env=env, capture_output=True, text=True, timeout=30, diff --git a/utils/test_synthetic_acceptance.py b/utils/test_synthetic_acceptance.py index ffa8b01aa0..a3ae8f92f2 100644 --- a/utils/test_synthetic_acceptance.py +++ b/utils/test_synthetic_acceptance.py @@ -267,6 +267,28 @@ def test_real_runs_clear_synthetic_without_a_curve( assert result["environment"]["KEEP"] == "yes" +@pytest.mark.parametrize( + ("prefix", "args", "expected"), + [ + ("dsv4", {"method": "dspark"}, 2.7), + ("kimik3", {"method": "dspark", "draft-model": "Inferact/Kimi-K3-DSpark"}, 2.9), + ("minimaxm3", {"method": "eagle3", "draft-model": "Inferact/MiniMax-M3-EAGLE3-GQA"}, 2.6), + ], +) +def test_atom_forces_golden_acceptance_by_server_flag( + golden_dir: Path, prefix: str, args: dict[str, Any], expected: float +) -> None: + recipe = {"roles": {"agg": {"args": {**args, "num-speculative-tokens": 3}, "env": {}}}} + env = {**ENV, "MODEL_PREFIX": prefix} + result = apply_native(recipe, build_overrides(recipe, "atom", env, golden_dir=golden_dir)) + assert result["roles"]["agg"]["args"]["spec-decode-acceptance-length"] == expected + # Evals verify real drafts, so a recipe-pinned acceptance length is removed. + evaluated = apply_native( + result, build_overrides(result, "atom", {**env, "EVAL_ONLY": "true"}, golden_dir=golden_dir) + ) + assert "spec-decode-acceptance-length" not in evaluated["roles"]["agg"]["args"] + + @pytest.mark.parametrize( "curve", [