diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b200.sh deleted file mode 100755 index 898402776a..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b200.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -# Common SGLANG env vars (apply to every config). -export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1 - -# TODO(Cam): the lmsysorg/sglang:deepseek-v4-blackwell image installs sglang -# editable at /workspace/sglang/python; prior sglang tags used /sgl-workspace/sglang. -# The runner mounts our repo at a non-/workspace path for this image so the editable -# install stays visible. Paths in this script are $PWD-relative for that reason. -# Drop the runner conditional once lmsys moves sglang back out of /workspace. - -SERVER_LOG="$PWD/server.log" -PORT=${PORT:-8888} - -echo "TP: $TP, DP_ATTENTION: $DP_ATTENTION, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -# 1k inputs need more SWA cache headroom than 8k inputs do. -if [[ "$ISL" == "1024" ]]; then - SWA_FULL_TOKENS_RATIO=0.5 -else - SWA_FULL_TOKENS_RATIO=0.1 -fi - -# Pick the parallelism + MoE backend based on DP_ATTENTION. DP-attention turns on -# EP-MoE (megamoe) + the mega_moe / mixed-chunk optimizations; single-instance -# uses flashinfer_mxfp4. -if [ "${DP_ATTENTION}" = "true" ]; then - export SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION=8 - export SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 - export SGLANG_OPT_USE_FAST_MASK_EP=1 - export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1 - export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1 - export NVSHMEM_DISABLE_IB=1 - export SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW=1 - export SGLANG_OPT_USE_ONLINE_COMPRESS=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=2048 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1 - export SGLANG_EXPERIMENTAL_ENABLE_PIECEWISE_CUDA_GRAPH_MOE_A2A=1 - export NCCL_MNNVL_ENABLE=1 - export NCCL_CUMEM_ENABLE=1 - export MC_FORCE_MNNVL=1 - export SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True - - MEM_FRACTION_STATIC=0.835 - MAX_RUNNING_REQUESTS=4352 - SWA_FULL_TOKENS_RATIO=0.12 - - PARALLEL_ARGS=( - --dp-size "$TP" - --enable-dp-attention - --moe-a2a-backend megamoe - --cuda-graph-max-bs 544 - --enable-mixed-chunk - --chunked-prefill-size 16384 - --max-prefill-tokens 16384 - --tokenizer-worker-num 8 - --stream-interval 30 - --enable-prefill-delayer - ) -else - MEM_FRACTION_STATIC=0.90 - MAX_RUNNING_REQUESTS=512 - PARALLEL_ARGS=( - --moe-runner-backend flashinfer_mxfp4 - --chunked-prefill-size 8192 - --disable-flashinfer-autotune - --cuda-graph-max-bs 512 - --tokenizer-worker-num 8 - --stream-interval 30 - --enable-prefill-delayer - ) -fi - -# Print all SGLANG_* env vars to both the CI step log and server.log so the -# launch config is auditable from the result artifact alone. -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" - -set -x -PYTHONNOUSERSITE=1 sglang serve \ - --model-path $MODEL \ - --host 0.0.0.0 \ - --port $PORT \ - --trust-remote-code \ - --tp $TP \ - --disable-radix-cache \ - --max-running-requests "$MAX_RUNNING_REQUESTS" \ - --mem-fraction-static "$MEM_FRACTION_STATIC" \ - --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" \ - "${PARALLEL_ARGS[@]}" $EVAL_CONTEXT_ARGS >> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $((CONC * 10)) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b200_trt.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b200_trt.sh deleted file mode 100644 index ce567c908e..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b200_trt.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro single-node TRTLLM recipe for B200. The configured image -# already contains a TensorRT-LLM DeepSeek-V4 build; do not build TRTLLM at -# runtime from this benchmark path. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -export TRTLLM_DSV4_USE_MPIRUN="${TRTLLM_DSV4_USE_MPIRUN:-1}" -export TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV="${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-1}" - -sanitize_slurm_mpi_env_for_trtllm() { - if [[ "${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-0}" != "1" ]]; then - return 0 - fi - - echo "Sanitizing Slurm/PMI environment for TensorRT-LLM launch" - while IFS='=' read -r name _; do - case "$name" in - SLURM_*|PMIX*|PMI*|OMPI_*|ORTE_*) - unset "$name" - ;; - esac - done < <(env) -} - -sanitize_slurm_mpi_env_for_trtllm - -export NCCL_NVLS_ENABLE="${NCCL_NVLS_ENABLE:-0}" -echo "NCCL_NVLS_ENABLE: $NCCL_NVLS_ENABLE" - -export TRTLLM_SERVER_DISABLE_GC="${TRTLLM_SERVER_DISABLE_GC:-1}" -export TRTLLM_WORKER_DISABLE_GC="${TRTLLM_WORKER_DISABLE_GC:-1}" -export NCCL_GRAPH_MIXING_SUPPORT="${NCCL_GRAPH_MIXING_SUPPORT:-0}" -export MIMALLOC_PURGE_DELAY="${MIMALLOC_PURGE_DELAY:-0}" -export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" - -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" -fi - -nvidia-smi - -SERVER_LOG="$PWD/server.log" -EXTRA_CONFIG_FILE="dsv4-fp4-trt.yml" - -# MoE backend: TRTLLM at low/mid concurrency; switch to MEGAMOE_DEEPGEMM at the -# top concurrency for short ISL (1k). -if [[ "$ISL" -le 1024 && "$CONC" -ge 2048 ]]; then - MOE_BACKEND="${MOE_BACKEND:-MEGAMOE_DEEPGEMM}" -else - MOE_BACKEND="${MOE_BACKEND:-TRTLLM}" -fi -MAX_BATCH_SIZE=$(( CONC > 16 ? CONC : 16 )) -CUDA_GRAPH_MAX_BATCH_SIZE="$MAX_BATCH_SIZE" -if [[ "$DP_ATTENTION" == "true" ]]; then - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.7}" -else - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.9}" -fi - -ATTENTION_DP_CONFIG="" -if [[ "$DP_ATTENTION" == "true" ]]; then - ATTENTION_DP_CONFIG=" -attention_dp_config: - batching_wait_iters: 30 - enable_balance: true" -fi - -cat > "$EXTRA_CONFIG_FILE" << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $CUDA_GRAPH_MAX_BATCH_SIZE -enable_attention_dp: $DP_ATTENTION$ATTENTION_DP_CONFIG -print_iter_log: true -kv_cache_config: - tokens_per_block: 128 - dtype: fp8 - free_gpu_memory_fraction: $KV_CACHE_FREE_MEM_FRACTION - enable_block_reuse: false -stream_interval: 100 -num_postprocess_workers: 4 -moe_config: - backend: $MOE_BACKEND - use_low_precision_moe_combine: true -EOF - -echo "Generated config file contents:" -cat "$EXTRA_CONFIG_FILE" - -MAX_MODEL_LEN=$(( MAX_MODEL_LEN > 8192 ? MAX_MODEL_LEN : 8192 )) -MAX_NUM_TOKENS=$(( ISL + 256 )) -MAX_NUM_TOKENS=$(( MAX_NUM_TOKENS > 8192 ? MAX_NUM_TOKENS : 8192 )) - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -set -x -SERVE_CMD=( - trtllm-serve "$MODEL" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --trust_remote_code \ - --backend pytorch \ - --max_batch_size "$MAX_BATCH_SIZE" \ - --max_seq_len "$MAX_MODEL_LEN" \ - --max_num_tokens "$MAX_NUM_TOKENS" \ - --tp_size "$TP" \ - --ep_size "$EP_SIZE" \ - --custom_tokenizer deepseek_v4 \ - --config "$EXTRA_CONFIG_FILE" -) - -if [[ "${TRTLLM_DSV4_USE_MPIRUN:-1}" == "0" ]]; then - "${SERVE_CMD[@]}" > "$SERVER_LOG" 2>&1 & -else - mpirun -n 1 --oversubscribe --allow-run-as-root \ - "${SERVE_CMD[@]}" \ - > "$SERVER_LOG" 2>&1 & -fi - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai-chat \ - --endpoint /v1/chat/completions \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$(( CONC * 10 ))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" \ - --trust-remote-code \ - --server-pid "$SERVER_PID" - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b200_trt_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b200_trt_mtp.sh deleted file mode 100644 index 0c7f323637..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b200_trt_mtp.sh +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro B200 TensorRT-LLM MTP variant. The configured image already -# contains the DeepSeek-V4 TRTLLM build; this path only toggles speculative MTP. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -export TRTLLM_DSV4_USE_MPIRUN="${TRTLLM_DSV4_USE_MPIRUN:-1}" -export TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV="${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-1}" - -sanitize_slurm_mpi_env_for_trtllm() { - if [[ "${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-0}" != "1" ]]; then - return 0 - fi - - echo "Sanitizing Slurm/PMI environment for TensorRT-LLM launch" - while IFS='=' read -r name _; do - case "$name" in - SLURM_*|PMIX*|PMI*|OMPI_*|ORTE_*) - unset "$name" - ;; - esac - done < <(env) -} - -sanitize_slurm_mpi_env_for_trtllm - -export NCCL_NVLS_ENABLE="${NCCL_NVLS_ENABLE:-0}" -echo "NCCL_NVLS_ENABLE: $NCCL_NVLS_ENABLE" - -export TRTLLM_SERVER_DISABLE_GC="${TRTLLM_SERVER_DISABLE_GC:-1}" -export TRTLLM_WORKER_DISABLE_GC="${TRTLLM_WORKER_DISABLE_GC:-1}" -export NCCL_GRAPH_MIXING_SUPPORT="${NCCL_GRAPH_MIXING_SUPPORT:-0}" -export MIMALLOC_PURGE_DELAY="${MIMALLOC_PURGE_DELAY:-0}" -export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" - -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" -fi - -nvidia-smi - -SERVER_LOG="$PWD/server.log" -EXTRA_CONFIG_FILE="dsv4-fp4-trt-mtp.yml" - -# MoE backend: TRTLLM at low/mid concurrency; switch to MEGAMOE_DEEPGEMM at high -# concurrency for short ISL (1k). -if [[ "$ISL" -le 1024 && "$CONC" -ge 512 ]]; then - MOE_BACKEND="${MOE_BACKEND:-MEGAMOE_DEEPGEMM}" -else - MOE_BACKEND="${MOE_BACKEND:-TRTLLM}" -fi -# MTP draft length: 3 at low/mid concurrency; steps down to 2 at high concurrency -# for long ISL (8k). -if [[ "$ISL" -ge 4096 && "$CONC" -ge 128 ]]; then - MTP="${TRTLLM_DSV4_MTP_NUM_NEXTN_LAYERS:-2}" -else - MTP="${TRTLLM_DSV4_MTP_NUM_NEXTN_LAYERS:-3}" -fi -MAX_BATCH_SIZE=$(( CONC > 16 ? CONC : 16 )) -CUDA_GRAPH_MAX_BATCH_SIZE="$MAX_BATCH_SIZE" -if [[ "$DP_ATTENTION" == "true" ]]; then - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.6}" -else - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.9}" -fi - -ATTENTION_DP_CONFIG="" -if [[ "$DP_ATTENTION" == "true" ]]; then - ATTENTION_DP_CONFIG=" -attention_dp_config: - batching_wait_iters: 30 - enable_balance: true -enable_lm_head_tp_in_adp: true" -fi - -cat > "$EXTRA_CONFIG_FILE" << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $CUDA_GRAPH_MAX_BATCH_SIZE -enable_attention_dp: $DP_ATTENTION$ATTENTION_DP_CONFIG -print_iter_log: true -kv_cache_config: - tokens_per_block: 128 - dtype: fp8 - free_gpu_memory_fraction: $KV_CACHE_FREE_MEM_FRACTION - enable_block_reuse: false -stream_interval: 100 -num_postprocess_workers: 4 -moe_config: - backend: $MOE_BACKEND - use_low_precision_moe_combine: true -speculative_config: - decoding_type: MTP - max_draft_len: $MTP -EOF - -echo "Generated config file contents:" -cat "$EXTRA_CONFIG_FILE" - -MAX_MODEL_LEN=$(( MAX_MODEL_LEN > 8192 ? MAX_MODEL_LEN : 8192 )) -MAX_NUM_TOKENS=$(( ISL + (MTP + 1) * MAX_BATCH_SIZE + 256 )) -MAX_NUM_TOKENS=$(( MAX_NUM_TOKENS > 8192 ? MAX_NUM_TOKENS : 8192 )) - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -set -x -SERVE_CMD=( - trtllm-serve "$MODEL" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --trust_remote_code \ - --backend pytorch \ - --max_batch_size "$MAX_BATCH_SIZE" \ - --max_seq_len "$MAX_MODEL_LEN" \ - --max_num_tokens "$MAX_NUM_TOKENS" \ - --tp_size "$TP" \ - --ep_size "$EP_SIZE" \ - --custom_tokenizer deepseek_v4 \ - --config "$EXTRA_CONFIG_FILE" -) - -if [[ "${TRTLLM_DSV4_USE_MPIRUN:-1}" == "0" ]]; then - "${SERVE_CMD[@]}" > "$SERVER_LOG" 2>&1 & -else - mpirun -n 1 --oversubscribe --allow-run-as-root \ - "${SERVE_CMD[@]}" \ - > "$SERVER_LOG" 2>&1 & -fi - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$(( CONC * 10 ))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" \ - --trust-remote-code \ - --use-chat-template \ - --dsv4 \ - --server-pid "$SERVER_PID" - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b200_vllm.sh deleted file mode 100755 index 3d174f205b..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b200_vllm.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro B200 single-node vLLM recipe derived from the B200 pareto -# sweep. TP mode (dp-attn=false) runs without expert parallel; DP mode -# (dp-attn=true) enables expert parallel (EP_SIZE=TP value = DP size). - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -# DeepSeek-V4-Pro weights are large; engine startup can exceed the default -# 600s. Give it an hour to load. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -MOE_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) - # The pinned checkpoint uses ModelOpt NVFP4 expert weights. Native - # DeepGEMM MegaMoE expects MXFP4 weights; use FlashInfer CuTeDSL instead. - MOE_ARGS=(--moe-backend flashinfer_cutedsl) -fi - -GMU_ARGS=() -PREFILL_SCHEDULE_ARGS=() -if [ "${DP_ATTENTION}" = "true" ]; then - # Keep EPLB disabled for the FlashInfer CuTeDSL path. - PREFILL_SCHEDULE_ARGS=(--prefill-schedule-interval 4) -fi - -if [ "${ISL}" -eq 8192 ] && [ "${CONC}" -le 128 ]; then - MAX_NUM_BATCHED_TOKENS=${ISL} -else - MAX_NUM_BATCHED_TOKENS=2048 -fi - -MAX_CUDAGRAPH_CAPTURE_SIZE=2048 - -BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN" -# Cap MAX_MODEL_LEN to free CUDA-graph memory and give more headroom for KV blocks -if [ "${BENCHMARK_MAX_MODEL_LEN}" -gt 12288 ]; then - BENCHMARK_MAX_MODEL_LEN=12288 -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - EVAL_MAX_MODEL_LEN=$(compute_eval_context_length "$MODEL" "$BENCHMARK_MAX_MODEL_LEN") - export EVAL_MAX_MODEL_LEN - SERVE_MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -else - SERVE_MAX_MODEL_LEN="$BENCHMARK_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve "$MODEL" --host 0.0.0.0 --port "$PORT" \ - --trust-remote-code \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --no-enable-prefix-caching \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - "${GMU_ARGS[@]}" \ - "${MOE_ARGS[@]}" \ - "${PREFILL_SCHEDULE_ARGS[@]}" \ - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ - --attention_config.use_fp4_indexer_cache=True \ - --tokenizer-mode deepseek_v4 \ - --tool-call-parser deepseek_v4 \ - --enable-auto-tool-choice \ - --reasoning-parser deepseek_v4 \ - --max-cudagraph-capture-size "$MAX_CUDAGRAPH_CAPTURE_SIZE" \ - --gpu-memory-utilization 0.95 \ - --max-model-len "$SERVE_MAX_MODEL_LEN" \ - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b200_vllm_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b200_vllm_mtp.sh deleted file mode 100755 index 2a1fa41454..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b200_vllm_mtp.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro B200 single-node vLLM MTP variant of dsv4_fp4_b200_vllm.sh. -# Adds --speculative-config '{"method":"mtp","num_speculative_tokens":2}' and -# routes prompts through chat-formatted encoding via --dsv4 (required for -# meaningful MTP acceptance numbers). - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -# DeepSeek-V4-Pro weights are large; engine startup can exceed the default -# 600s. Give it an hour to load. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -# Mega-MoE backend and the lower GMU only kick in on the DP-attn path, -# per the vLLM v0.20.0 DeepSeek-V4-Pro recipe. All configs share the -# FULL_AND_PIECEWISE compilation config. -GMU_ARGS=() -MOE_ARGS=() -PREFILL_SCHEDULE_ARGS=() -if [ "${DP_ATTENTION}" = "true" ]; then - GMU_ARGS=(--gpu-memory-utilization 0.9) - MOE_ARGS=(--moe-backend deep_gemm_mega_moe) - PREFILL_SCHEDULE_ARGS=(--prefill-schedule-interval 4) -fi - -MAX_NUM_BATCHED_TOKENS=$(( ISL * 2 )) -BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN" - -if [ "${EVAL_ONLY}" = "true" ]; then - EVAL_MAX_MODEL_LEN=$(compute_eval_context_length "$MODEL" "$BENCHMARK_MAX_MODEL_LEN") - export EVAL_MAX_MODEL_LEN - SERVE_MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -else - SERVE_MAX_MODEL_LEN="$BENCHMARK_MAX_MODEL_LEN" -fi - -# use 2 speculative tokens for all configs for now -NUM_SPEC_TOKENS=2 - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve "$MODEL" --host 0.0.0.0 --port "$PORT" \ - --trust-remote-code \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --no-enable-prefix-caching \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - "${GMU_ARGS[@]}" \ - "${MOE_ARGS[@]}" \ - "${PREFILL_SCHEDULE_ARGS[@]}" \ - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ - --attention_config.use_fp4_indexer_cache=True \ - --tokenizer-mode deepseek_v4 \ - --tool-call-parser deepseek_v4 \ - --enable-auto-tool-choice \ - --reasoning-parser deepseek_v4 \ - --max-cudagraph-capture-size 2048 \ - --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ - --max-model-len "$SERVE_MAX_MODEL_LEN" \ - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# MTP acceptance rate degrades on raw random tokens; --dsv4 routes prompts -# through chat-formatted encoding as required for speculative decoding benchmarks. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --dsv4 - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_sglang.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_sglang.sh deleted file mode 100755 index b451dee0d5..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_sglang.sh +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# ─── Common env vars (all profiles) ─────────────────────────────────────────── -export SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 -export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1 - -SERVER_LOG="$PWD/server.log" - -echo "TP: $TP, DP_ATTENTION: $DP_ATTENTION, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -# ─── Per-concurrency launch profile ────────────────────────────────────────── -# Each block sets: PARALLEL_ARGS, MEM_FRACTION_STATIC, SWA_FULL_TOKENS_RATIO, -# and optionally MAX_RUNNING_REQUESTS plus profile-specific env vars. -# -# SWA ratio: 1k inputs need more SWA cache headroom than 8k inputs; 0.5 was -# tuned empirically for the 1k1k recipe, while 0.1 is the cookbook default. - -if [ "$CONC" = "1" ] || [ "$CONC" = "32" ]; then - # TP-only, no DP attention - MEM_FRACTION_STATIC=0.90 - SWA_FULL_TOKENS_RATIO=$([[ "$ISL" == "1024" ]] && echo 0.5 || echo 0.1) - PARALLEL_ARGS=( - --moe-runner-backend flashinfer_mxfp4 - --chunked-prefill-size 8192 - --disable-flashinfer-autotune - ) - -elif [ "$CONC" = "512" ]; then - # DP attention, flashinfer_mxfp4 - export SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 - MEM_FRACTION_STATIC=0.94 - SWA_FULL_TOKENS_RATIO=$([[ "$ISL" == "1024" ]] && echo 0.5 || echo 0.1) - PARALLEL_ARGS=( - --dp-size "$TP" - --enable-dp-attention - --moe-runner-backend flashinfer_mxfp4 - --disable-flashinfer-autotune - --chunked-prefill-size 16384 - --enable-prefill-delayer - ) - -elif [ "$CONC" = "2048" ]; then - # DP attention, megamoe - export SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 - export NVSHMEM_DISABLE_IB=1 - export SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW=1 - export SGLANG_LOG_FORWARD_ITERS=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 - MEM_FRACTION_STATIC=0.87 - SWA_FULL_TOKENS_RATIO=0.06 - MAX_RUNNING_REQUESTS=2560 - PARALLEL_ARGS=( - --dp-size "$TP" - --enable-dp-attention - --moe-a2a-backend megamoe - --cuda-graph-max-bs 288 - --chunked-prefill-size 65536 - --tokenizer-worker-num 4 - --enable-prefill-delayer - ) - -elif [ "$CONC" = "4096" ]; then - # DP attention, megamoe - export SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 - export NVSHMEM_DISABLE_IB=1 - export SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 - MEM_FRACTION_STATIC=0.835 - SWA_FULL_TOKENS_RATIO=0.075 - MAX_RUNNING_REQUESTS=4352 - PARALLEL_ARGS=( - --dp-size "$TP" - --enable-dp-attention - --moe-a2a-backend megamoe - --cuda-graph-max-bs 544 - --chunked-prefill-size 65536 - --tokenizer-worker-num 8 - --enable-prefill-delayer - --decode-log-interval 5 - ) - -elif [ "$CONC" = "8192" ]; then - # DP attention, megamoe - export SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 - export NVSHMEM_DISABLE_IB=1 - export SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW=1 - export SGLANG_OPT_USE_ONLINE_COMPRESS=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8256 - MEM_FRACTION_STATIC=0.80 - SWA_FULL_TOKENS_RATIO=0.3 - MAX_RUNNING_REQUESTS=8192 - PARALLEL_ARGS=( - --dp-size "$TP" - --enable-dp-attention - --moe-a2a-backend megamoe - --cuda-graph-max-bs 1088 - --chunked-prefill-size 65536 - --tokenizer-worker-num 16 - --enable-prefill-delayer - --stream-interval 30 - ) - -else - echo "ERROR: unsupported CONC=$CONC" >&2 - exit 1 -fi - -# Print all SGLANG_* env vars to both the CI step log and server.log so the -# launch config is auditable from the result artifact alone. -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" - -set -x -PYTHONNOUSERSITE=1 sglang serve \ - --model-path $MODEL_PATH --served-model-name $MODEL \ - --host 0.0.0.0 \ - --port $PORT \ - --trust-remote-code \ - --tp $TP \ - --max-running-requests "${MAX_RUNNING_REQUESTS:-$(( CONC * 3 / 2 > 8 ? CONC * 3 / 2 : 8 ))}" \ - --mem-fraction-static "$MEM_FRACTION_STATIC" \ - --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" \ - "${PARALLEL_ARGS[@]}" $EVAL_CONTEXT_ARGS >> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas -pip install -q --upgrade transformers - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $((CONC * 10)) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_sglang_mtp.sh deleted file mode 100755 index 9c8bab961b..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_sglang_mtp.sh +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -# Tuning inputs from the matrix (all required): -# TP -- tensor parallel size -> --tp -# EP_SIZE -- expert parallel size -> --ep-size -# DP_ATTENTION -- "true" enables --enable-dp-attention --dp-size $TP -# Also selects MoE backend / chunked-prefill / EAGLE chain -# / mem-fraction-static / max-running-requests: -# true -> flashinfer_mxfp4 + DP-attn + chunked-prefill 32768 -# + EAGLE (1,1,2) + mem-fraction 0.92 + max-running 256 -# false -> flashinfer_mxfp4 (TP-only) + chunked-prefill 8192 -# + EAGLE (3,1,4) + mem-fraction 0.90 + max-running CONC*3/2 -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# Common SGLANG env vars. -export SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 -export SGLANG_RADIX_FORCE_MISS=1 -export SGLANG_DEFAULT_THINKING=1 -export SGLANG_DSV4_REASONING_EFFORT=max -export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1 - -# TODO(Cam): the deepseek-v4 sglang images install sglang editable at -# /workspace/sglang/python; prior sglang tags used /sgl-workspace/sglang. -# The runner mounts our repo at a non-/workspace path for these images so the -# editable install stays visible. Paths in this script are $PWD-relative for -# that reason. Drop the runner conditional once lmsys moves sglang back out of -# /workspace. - -SERVER_LOG="$PWD/server.log" - -echo "TP: $TP, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -# Recipe path is selected by DP_ATTENTION; MoE backend, chunked-prefill, EAGLE -# chain, mem-fraction, and max-running all follow. -DEEPEP_CONFIG='{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - -if [ "${DP_ATTENTION}" = "true" ]; then - # DP-attn path: flashinfer_mxfp4 + DP-attn (covers conc 16-256). - export SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 - export SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8192 - export SGLANG_REQUEST_STATE_WAIT_TIMEOUT=60 - SPEC_FLAGS=( - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 - ) - PARALLEL_ARGS=( - --dp-size "$TP" - --enable-dp-attention - --moe-runner-backend flashinfer_mxfp4 - --disable-flashinfer-autotune - --deepep-config "$DEEPEP_CONFIG" - --cuda-graph-max-bs 256 - --enable-deepseek-v4-fp4-indexer - ) - CHUNKED_PREFILL_SIZE=32768 - MEM_FRACTION_STATIC=0.92 - MAX_RUNNING_REQUESTS=256 -else - # TP-only fallback for low-conc: flashinfer_mxfp4 + EAGLE (3,1,4). - SPEC_FLAGS=( - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 - ) - PARALLEL_ARGS=( - --moe-runner-backend flashinfer_mxfp4 - --disable-flashinfer-autotune - --enable-deepseek-v4-fp4-indexer - ) - CHUNKED_PREFILL_SIZE=8192 - MEM_FRACTION_STATIC=0.90 - MAX_RUNNING_REQUESTS="$(( CONC * 3 / 2 > 8 ? CONC * 3 / 2 : 8 ))" -fi - -# Print all SGLANG_* env vars to both the CI step log and server.log so the -# launch config is auditable from the result artifact alone. -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" - -set -x -PYTHONNOUSERSITE=1 sglang serve \ - --model-path $MODEL_PATH --served-model-name $MODEL \ - --host 0.0.0.0 \ - --port $PORT \ - --trust-remote-code \ - --tp $TP \ - --ep-size $EP_SIZE \ - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" \ - --max-running-requests "$MAX_RUNNING_REQUESTS" \ - --mem-fraction-static "$MEM_FRACTION_STATIC" \ - --swa-full-tokens-ratio 0.1 \ - "${SPEC_FLAGS[@]}" \ - "${PARALLEL_ARGS[@]}" $EVAL_CONTEXT_ARGS >> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# --dsv4 routes prompts through encoding_dsv4.py (PR #1153), which emits the -# ... framing DeepSeek-V4-Pro expects. The DSv4-Pro -# tokenizer ships without a jinja chat_template, so plain --use-chat-template -# would crash; --dsv4 sidesteps that and satisfies the AGENTS.md rule that all -# MTP scripts must benchmark against chat-formatted inputs (EAGLE acceptance -# silently regresses on raw random tokens). -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $((CONC * 10)) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" \ - --dsv4 - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_trt.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_trt.sh deleted file mode 100644 index bcd1fbf6ae..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_trt.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro single-node TRTLLM recipe for B300. The configured image -# already contains a TensorRT-LLM DeepSeek-V4 build; do not build TRTLLM at -# runtime from this benchmark path. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -export TRTLLM_DSV4_USE_MPIRUN="${TRTLLM_DSV4_USE_MPIRUN:-1}" -export TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV="${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-1}" - -sanitize_slurm_mpi_env_for_trtllm() { - if [[ "${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-0}" != "1" ]]; then - return 0 - fi - - echo "Sanitizing Slurm/PMI environment for TensorRT-LLM launch" - while IFS='=' read -r name _; do - case "$name" in - SLURM_*|PMIX*|PMI*|OMPI_*|ORTE_*) - unset "$name" - ;; - esac - done < <(env) -} - -sanitize_slurm_mpi_env_for_trtllm - -export NCCL_NVLS_ENABLE="${NCCL_NVLS_ENABLE:-0}" -echo "NCCL_NVLS_ENABLE: $NCCL_NVLS_ENABLE" - -export TRTLLM_SERVER_DISABLE_GC="${TRTLLM_SERVER_DISABLE_GC:-1}" -export TRTLLM_WORKER_DISABLE_GC="${TRTLLM_WORKER_DISABLE_GC:-1}" -export NCCL_GRAPH_MIXING_SUPPORT="${NCCL_GRAPH_MIXING_SUPPORT:-0}" -export MIMALLOC_PURGE_DELAY="${MIMALLOC_PURGE_DELAY:-0}" -export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" - -nvidia-smi - -SERVER_LOG="$PWD/server.log" -EXTRA_CONFIG_FILE="dsv4-fp4-trt.yml" - -# MoE backend: TRTLLM at low/mid concurrency; switch to MEGAMOE_DEEPGEMM at the -# top concurrency for short ISL (1k). -if [[ "$ISL" -le 1024 && "$CONC" -ge 2048 ]]; then - MOE_BACKEND="${MOE_BACKEND:-MEGAMOE_DEEPGEMM}" -else - MOE_BACKEND="${MOE_BACKEND:-TRTLLM}" -fi -MAX_BATCH_SIZE=$(( CONC > 16 ? CONC : 16 )) -# Cap CUDA-graph capture at batch 1024. TRTLLM_MLA_EXTRA_OVERLAP hands MLA -# prologue tensors across streams without record_stream(), so graph warmup at -# decode batch >1024 (repros at 1088, e.g. tp8/ep8 dp-attn conc-2048 on B300) -# hits a use-after-free -> CUDA_ERROR_ILLEGAL_ADDRESS. Fixed upstream in -# NVIDIA/TensorRT-LLM#15265; cap until that fix ships in the image. Runtime -# --max_batch_size stays = CONC, so batches >1024 just run eager. -CUDA_GRAPH_MAX_BATCH_SIZE=$(( MAX_BATCH_SIZE < 1024 ? MAX_BATCH_SIZE : 1024 )) -if [[ "$DP_ATTENTION" == "true" ]]; then - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.7}" -else - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.9}" -fi - -ATTENTION_DP_CONFIG="" -if [[ "$DP_ATTENTION" == "true" ]]; then - ATTENTION_DP_CONFIG=" -attention_dp_config: - batching_wait_iters: 30 - enable_balance: true" -fi - -cat > "$EXTRA_CONFIG_FILE" << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $CUDA_GRAPH_MAX_BATCH_SIZE -enable_attention_dp: $DP_ATTENTION$ATTENTION_DP_CONFIG -print_iter_log: true -kv_cache_config: - tokens_per_block: 128 - dtype: fp8 - free_gpu_memory_fraction: $KV_CACHE_FREE_MEM_FRACTION - enable_block_reuse: false -stream_interval: 100 -num_postprocess_workers: 4 -moe_config: - backend: $MOE_BACKEND - use_low_precision_moe_combine: true -EOF - -echo "Generated config file contents:" -cat "$EXTRA_CONFIG_FILE" - -MAX_MODEL_LEN=$(( MAX_MODEL_LEN > 8192 ? MAX_MODEL_LEN : 8192 )) -MAX_NUM_TOKENS=$(( ISL + 256 )) -MAX_NUM_TOKENS=$(( MAX_NUM_TOKENS > 8192 ? MAX_NUM_TOKENS : 8192 )) - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -export TRTLLM_MHC_ENABLE_FUSED_HC="${TRTLLM_MHC_ENABLE_FUSED_HC:-1}" -echo "TRTLLM_MHC_ENABLE_FUSED_HC: $TRTLLM_MHC_ENABLE_FUSED_HC" - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -set -x -SERVE_CMD=( - trtllm-serve "$MODEL_PATH" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --trust_remote_code \ - --backend pytorch \ - --max_batch_size "$MAX_BATCH_SIZE" \ - --max_seq_len "$MAX_MODEL_LEN" \ - --max_num_tokens "$MAX_NUM_TOKENS" \ - --tp_size "$TP" \ - --ep_size "$EP_SIZE" \ - --custom_tokenizer deepseek_v4 \ - --config "$EXTRA_CONFIG_FILE" -) - -if [[ "${TRTLLM_DSV4_USE_MPIRUN:-1}" == "0" ]]; then - "${SERVE_CMD[@]}" > "$SERVER_LOG" 2>&1 & -else - mpirun -n 1 --oversubscribe --allow-run-as-root \ - "${SERVE_CMD[@]}" \ - > "$SERVER_LOG" 2>&1 & -fi - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai-chat \ - --endpoint /v1/chat/completions \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$(( CONC * 10 ))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" \ - --trust-remote-code \ - --server-pid "$SERVER_PID" - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_trt_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_trt_mtp.sh deleted file mode 100644 index bb0362c256..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_trt_mtp.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro B300 TensorRT-LLM MTP variant. The configured image already -# contains the DeepSeek-V4 TRTLLM build; this path only toggles speculative MTP. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -export TRTLLM_DSV4_USE_MPIRUN="${TRTLLM_DSV4_USE_MPIRUN:-1}" -export TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV="${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-1}" - -sanitize_slurm_mpi_env_for_trtllm() { - if [[ "${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-0}" != "1" ]]; then - return 0 - fi - - echo "Sanitizing Slurm/PMI environment for TensorRT-LLM launch" - while IFS='=' read -r name _; do - case "$name" in - SLURM_*|PMIX*|PMI*|OMPI_*|ORTE_*) - unset "$name" - ;; - esac - done < <(env) -} - -sanitize_slurm_mpi_env_for_trtllm - -export NCCL_NVLS_ENABLE="${NCCL_NVLS_ENABLE:-0}" -echo "NCCL_NVLS_ENABLE: $NCCL_NVLS_ENABLE" - -export TRTLLM_SERVER_DISABLE_GC="${TRTLLM_SERVER_DISABLE_GC:-1}" -export TRTLLM_WORKER_DISABLE_GC="${TRTLLM_WORKER_DISABLE_GC:-1}" -export NCCL_GRAPH_MIXING_SUPPORT="${NCCL_GRAPH_MIXING_SUPPORT:-0}" -export MIMALLOC_PURGE_DELAY="${MIMALLOC_PURGE_DELAY:-0}" -export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" - -nvidia-smi - -SERVER_LOG="$PWD/server.log" -EXTRA_CONFIG_FILE="dsv4-fp4-trt-mtp.yml" - -# MoE backend: TRTLLM at low/mid concurrency; switch to MEGAMOE_DEEPGEMM at high -# concurrency for short ISL (1k). -if [[ "$ISL" -le 1024 && "$CONC" -ge 512 ]]; then - MOE_BACKEND="${MOE_BACKEND:-MEGAMOE_DEEPGEMM}" -else - MOE_BACKEND="${MOE_BACKEND:-TRTLLM}" -fi -# MTP draft length: 3 at low/mid concurrency; steps down to 2 at high concurrency -# for long ISL (8k). -if [[ "$ISL" -ge 4096 && "$CONC" -ge 128 ]]; then - MTP="${TRTLLM_DSV4_MTP_NUM_NEXTN_LAYERS:-2}" -else - MTP="${TRTLLM_DSV4_MTP_NUM_NEXTN_LAYERS:-3}" -fi -MAX_BATCH_SIZE=$(( CONC > 16 ? CONC : 16 )) -# Cap CUDA-graph capture at batch 1024. TRTLLM_MLA_EXTRA_OVERLAP hands MLA -# prologue tensors across streams without record_stream(), so graph warmup at -# decode batch >1024 (repros at 1088, e.g. tp8/ep8 dp-attn conc-2048 on B300) -# hits a use-after-free -> CUDA_ERROR_ILLEGAL_ADDRESS. Fixed upstream in -# NVIDIA/TensorRT-LLM#15265; cap until that fix ships in the image. Runtime -# --max_batch_size stays = CONC, so batches >1024 just run eager. -CUDA_GRAPH_MAX_BATCH_SIZE=$(( MAX_BATCH_SIZE < 1024 ? MAX_BATCH_SIZE : 1024 )) -if [[ "$DP_ATTENTION" == "true" ]]; then - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.6}" -else - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.9}" -fi - -ATTENTION_DP_CONFIG="" -if [[ "$DP_ATTENTION" == "true" ]]; then - ATTENTION_DP_CONFIG=" -attention_dp_config: - batching_wait_iters: 30 - enable_balance: true -enable_lm_head_tp_in_adp: true" -fi - -cat > "$EXTRA_CONFIG_FILE" << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $CUDA_GRAPH_MAX_BATCH_SIZE -enable_attention_dp: $DP_ATTENTION$ATTENTION_DP_CONFIG -print_iter_log: true -kv_cache_config: - tokens_per_block: 128 - dtype: fp8 - free_gpu_memory_fraction: $KV_CACHE_FREE_MEM_FRACTION - enable_block_reuse: false -stream_interval: 100 -num_postprocess_workers: 4 -moe_config: - backend: $MOE_BACKEND - use_low_precision_moe_combine: true -speculative_config: - decoding_type: MTP - max_draft_len: $MTP -EOF - -echo "Generated config file contents:" -cat "$EXTRA_CONFIG_FILE" - -MAX_MODEL_LEN=$(( MAX_MODEL_LEN > 8192 ? MAX_MODEL_LEN : 8192 )) -MAX_NUM_TOKENS=$(( ISL + (MTP + 1) * MAX_BATCH_SIZE + 256 )) -MAX_NUM_TOKENS=$(( MAX_NUM_TOKENS > 8192 ? MAX_NUM_TOKENS : 8192 )) - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -export TRTLLM_MHC_ENABLE_FUSED_HC="${TRTLLM_MHC_ENABLE_FUSED_HC:-1}" -echo "TRTLLM_MHC_ENABLE_FUSED_HC: $TRTLLM_MHC_ENABLE_FUSED_HC" - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -set -x -SERVE_CMD=( - trtllm-serve "$MODEL_PATH" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --trust_remote_code \ - --backend pytorch \ - --max_batch_size "$MAX_BATCH_SIZE" \ - --max_seq_len "$MAX_MODEL_LEN" \ - --max_num_tokens "$MAX_NUM_TOKENS" \ - --tp_size "$TP" \ - --ep_size "$EP_SIZE" \ - --custom_tokenizer deepseek_v4 \ - --config "$EXTRA_CONFIG_FILE" -) - -if [[ "${TRTLLM_DSV4_USE_MPIRUN:-1}" == "0" ]]; then - "${SERVE_CMD[@]}" > "$SERVER_LOG" 2>&1 & -else - mpirun -n 1 --oversubscribe --allow-run-as-root \ - "${SERVE_CMD[@]}" \ - > "$SERVER_LOG" 2>&1 & -fi - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$(( CONC * 10 ))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" \ - --trust-remote-code \ - --use-chat-template \ - --dsv4 \ - --server-pid "$SERVER_PID" - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_vllm.sh deleted file mode 100755 index 8aef70a8e0..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_vllm.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro B300 single-node aggregate recipe from the submitted B300 -# pareto sweep. TP mode (dp-attn=false) runs without expert parallel; DP mode -# (dp-attn=true) enables expert parallel (EP_SIZE=TP value = DP size). - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -SERVER_LOG=/workspace/server.log - -# DeepSeek-V4-Pro weights are large; engine startup can exceed the default -# 600s. Give it an hour to load. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -MOE_ARGS=() -if [ "${DP_ATTENTION}" = "true" ]; then - MOE_ARGS=(--moe-backend deep_gemm_mega_moe) -fi - -if [ "${DP_ATTENTION}" = "true" ]; then - MAX_NUM_BATCHED_TOKENS=2048 -else - MAX_NUM_BATCHED_TOKENS=$(( ISL * 2 )) -fi - -BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN" - -if [ "${EVAL_ONLY}" = "true" ]; then - EVAL_MAX_MODEL_LEN=$(compute_eval_context_length "$MODEL" "$BENCHMARK_MAX_MODEL_LEN") - export EVAL_MAX_MODEL_LEN - SERVE_MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -else - SERVE_MAX_MODEL_LEN="$BENCHMARK_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --pipeline-parallel-size 1 \ - --kv-cache-dtype fp8 \ - --trust-remote-code \ - --block-size 256 \ - --no-enable-prefix-caching \ - "${EP_ARGS[@]}" \ - "${MOE_ARGS[@]}" \ - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ - --attention_config.use_fp4_indexer_cache True \ - --tokenizer-mode deepseek_v4 \ - --tool-call-parser deepseek_v4 \ - --enable-auto-tool-choice \ - --reasoning-parser deepseek_v4 \ - --max-cudagraph-capture-size 2048 \ - --max-model-len "$SERVE_MAX_MODEL_LEN" \ - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_vllm_mtp.sh deleted file mode 100755 index a5e7dd28cb..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_b300_vllm_mtp.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -SERVER_LOG=/workspace/server.log - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -MOE_ARGS=() -if [ "${DP_ATTENTION}" = "true" ]; then - MOE_ARGS=(--moe-backend deep_gemm_mega_moe) - MAX_NUM_BATCHED_TOKENS=2048 -else - MAX_NUM_BATCHED_TOKENS=$(( ISL * 2 )) -fi - -BENCHMARK_MAX_MODEL_LEN=$MAX_MODEL_LEN - -if [ "${EVAL_ONLY}" = "true" ]; then - EVAL_MAX_MODEL_LEN=$(compute_eval_context_length "$MODEL" "$BENCHMARK_MAX_MODEL_LEN") - export EVAL_MAX_MODEL_LEN - SERVE_MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -else - SERVE_MAX_MODEL_LEN="$BENCHMARK_MAX_MODEL_LEN" -fi - -# use 2 speculative tokens for all configs for now -NUM_SPEC_TOKENS=2 - -start_gpu_monitor - -set -x -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --pipeline-parallel-size 1 \ - --kv-cache-dtype fp8 \ - --trust-remote-code \ - --block-size 256 \ - --no-enable-prefix-caching \ - "${EP_ARGS[@]}" \ - "${MOE_ARGS[@]}" \ - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ - --attention_config.use_fp4_indexer_cache True \ - --tokenizer-mode deepseek_v4 \ - --tool-call-parser deepseek_v4 \ - --enable-auto-tool-choice \ - --reasoning-parser deepseek_v4 \ - --max-cudagraph-capture-size 2048 \ - --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ - --max-model-len "$SERVE_MAX_MODEL_LEN" \ - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# MTP acceptance rate degrades on raw random tokens; --dsv4 routes prompts -# through chat-formatted encoding as required for speculative decoding benchmarks. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --dsv4 - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_atom.sh deleted file mode 100644 index 2548978b15..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_atom.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -PARALLEL_ARGS=(-tp "$TP") #TP -CUDAGRAPH_SIZES='[1, 2, 4, 8, 16, 32, 48, 64, 128, 256, 512]' -if [ "$DP_ATTENTION" = "true" ]; then - if [ "$EP_SIZE" -gt 1 ]; then #DP+EP - PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) - else #DPA+TP - #DPA+TP+TBO (opt: TBO on for dp-attn cells at conc>=64, no per-scenario gate) - if [ "$CONC" -ge 64 ]; then - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention --enable-tbo) - export GPU_MAX_HW_QUEUES=5 - else - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) - fi - fi -fi - -# max_req=conc for every dp-on cell (mandatory: dp-attention keeps a full KV pool -# per rank, so the large default max_num_seqs OOMs even at low conc like c16/c32) -# and for mid/high conc (conc>=64). dp-off low conc uses the ATOM default -# (dev: default is on-par or ~4% better at very low conc, e.g. c2). -if [ "$DP_ATTENTION" = "true" ] || [ "$CONC" -ge 64 ]; then - PARALLEL_ARGS+=(--max-num-seqs "$CONC") -fi - -BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN" - -if [ "${EVAL_ONLY}" = "true" ]; then - EVAL_MAX_MODEL_LEN=$(compute_eval_context_length "$MODEL" "$BENCHMARK_MAX_MODEL_LEN") - export EVAL_MAX_MODEL_LEN -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -export ATOM_DISABLE_MMAP=true -export AITER_BF16_FP8_MOE_BOUND=0 -export ATOM_MOE_GU_ITLV=1 -MEM_FRAC_STATIC=0.9 - -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - "${PARALLEL_ARGS[@]}" \ - --kv_cache_dtype fp8 \ - --trust-remote-code \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --no-enable_prefix_caching \ - --cudagraph-capture-sizes "${CUDAGRAPH_SIZES}" \ - > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_atom_mtp.sh deleted file mode 100755 index 55ffc501b7..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_atom_mtp.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log -PORT=${PORT:-8888} - -PARALLEL_ARGS=(-tp "$TP") #TP -if [ "$DP_ATTENTION" = "true" ]; then - if [ "$EP_SIZE" -gt 1 ]; then #DP+EP - PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) - else #DPA+TP - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) - fi -fi - -# MTP speculative decoding (ATOM self-draft). -SPEC_ARGS=(--method mtp --num-speculative-tokens 3) - -# max_req=conc for dp-on cells (dp-attention keeps a full KV pool per rank, and MTP -# reserves q=num_speculative_tokens+1 per request, so the large default max_num_seqs -# OOMs) and for conc>=64. dp-off low conc uses the ATOM default. -if [ "$DP_ATTENTION" = "true" ] || [ "$CONC" -ge 64 ]; then - PARALLEL_ARGS+=(--max-num-seqs "$CONC") -fi - -# prefill-only TBO (--enable-tbo -> argparse const=prefill -> enable_tbo_decode=False): -# MTP-compatible because it never touches decode. (--enable-tbo all / decode-TBO WOULD -# drop MTP's spec_decode_metadata in UBatchWrapper -- that is the real incompatibility; -# prefill-only does not.) Threshold is conc>=256 for MTP (NOT the non-MTP #2327 conc>=64): -# measured crossover (TBO run 30257759947 vs non-TBO run 30238071409, same image/harness) -# shows TBO HURTS at c64/c128 (-14%/-10% output tput) but HELPS at c256+ (+11/+8/+14%). -# MTP already removes low-conc latency, so TBO overlap there is pure overhead; the win -# only appears once high conc turns compute-bound. Matches ATOM models.json non-MTP -# DPA-TBO 256-2048 band. Together with GPU_MAX_HW_QUEUES=5. -if [ "$DP_ATTENTION" = "true" ] && [ "$CONC" -ge 256 ]; then - PARALLEL_ARGS+=(--enable-tbo) - export GPU_MAX_HW_QUEUES=5 -fi - -BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN" - -if [ "${EVAL_ONLY}" = "true" ]; then - EVAL_MAX_MODEL_LEN=$(compute_eval_context_length "$MODEL" "$BENCHMARK_MAX_MODEL_LEN") - export EVAL_MAX_MODEL_LEN -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -export ATOM_DISABLE_MMAP=true -export AITER_BF16_FP8_MOE_BOUND=0 -export ATOM_MOE_GU_ITLV=1 - -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${SPEC_ARGS[@]}" \ - --kv_cache_dtype fp8 \ - --trust-remote-code \ - --no-enable_prefix_caching \ - > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# --dsv4: InferenceX's bench (utils/bench_serving/benchmark_serving.py) uses the -# DeepSeek-V4 chat encoder (encoding_dsv4.py) instead of the tokenizer's jinja -# template. DSv4-Pro has NO jinja chat_template, so a plain --use-chat-template -# produces empty/broken prompts (bench yields no result). --dsv4 implies -# --use-chat-template. Chat-formatted inputs are required for MTP: EAGLE/MTP -# acceptance silently regresses on raw random tokens (AGENTS.md). -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --dsv4 - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_sglang.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_sglang.sh deleted file mode 100755 index 79c5832db9..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_sglang.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - MAX_MODEL_LEN - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# sglang ships in the image at the SHA encoded in the image tag (built -# from the amd/deepseek_v4 branch in sgl-project/sglang). To bump sglang, -# bump the image tag in configs/amd-master.yaml. - -export SGLANG_DEFAULT_THINKING=1 -export SGLANG_DSV4_REASONING_EFFORT=max -export SGLANG_USE_ROCM700A=0 -export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton -export AITER_BF16_FP8_MOE_BOUND=0 - -SERVER_LOG=/workspace/server.log - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -PARALLEL_ARGS=( - --tensor-parallel-size "$TP" -) -CHUNKED_PREFILL_SIZE=$ISL -if [ "${DP_ATTENTION}" = "true" ]; then - export SGLANG_SHARED_EXPERT_TP1=1 - export SGLANG_DP_SHARED_EXPERT_LOCAL=1 - export SGLANG_DP_USE_GATHERV=1 - export SGLANG_DP_USE_REDUCE_SCATTER=1 - export GPU_MAX_HW_QUEUES=5 - - CHUNKED_PREFILL_SIZE=$((ISL * TP)) - PARALLEL_ARGS+=( - --dp "$TP" - --enable-dp-attention - --enable-prefill-delayer - --enable-two-batch-overlap - ) -fi -if [ "${EP_SIZE:-1}" -gt 1 ]; then - PARALLEL_ARGS+=(--ep-size "$EP_SIZE") -fi - -sglang serve \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - "${PARALLEL_ARGS[@]}" \ - --trust-remote-code \ - --disable-radix-cache \ - --attention-backend dsv4 \ - --cuda-graph-max-bs ${CONC} \ - --max-running-requests ${CONC} \ - --mem-fraction-static 0.90 \ - --swa-full-tokens-ratio 0.15 \ - --page-size 256 \ - --kv-cache-dtype fp8_e4m3 \ - --context-length $MAX_MODEL_LEN \ - --chunked-prefill-size $CHUNKED_PREFILL_SIZE \ - --disable-shared-experts-fusion \ - --tool-call-parser deepseekv4 \ - --reasoning-parser deepseek-v4 \ - --chat-template "$(dirname "$0")/../chat_templates/deepseek_v4_thinking.jinja" \ - --watchdog-timeout 1800 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_sglang_mtp.sh deleted file mode 100755 index 5e40a4d348..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_sglang_mtp.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro on MI355X via SGLang — MTP variant of dsv4_fp4_mi355x_sglang.sh. -# Adds EAGLE/MTP speculative decoding per sgl-project/sglang#26383 -# ([AMD][DSV4] DSV4 MTP graph + sparse triton attn optimizations, merged -# 2026-05-27, commit deaba74), which fixes the ROCm HIP-radix backend's -# per-step draft out_cache_loc slicing under CUDA graph (the bug behind the -# false-EOS / truncated-generation symptom in sgl issue #20404) and validates -# GSM8K 0.950 with MTP on. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - MAX_MODEL_LEN - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# sglang ships in the image at the SHA encoded in the image tag (built -# from the amd/deepseek_v4 branch in sgl-project/sglang). To bump sglang, -# bump the image tag in configs/amd-master.yaml. - -export SGLANG_DEFAULT_THINKING=1 -export SGLANG_DSV4_REASONING_EFFORT=max -export SGLANG_USE_ROCM700A=0 -export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton -export AITER_BF16_FP8_MOE_BOUND=0 -export TORCH_BLAS_PREFER_HIPBLASLT=1 -export SGLANG_OPT_USE_AITER_BATCHED_GEMM=1 - - -SERVER_LOG=/workspace/server.log -PORT=${PORT:-8888} - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -PARALLEL_ARGS=( - --tensor-parallel-size "$TP" -) -SPEC_FLAGS=( - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 -) -SHARED_EXPERTS_ARGS=(--enforce-shared-experts-fusion) -CHUNKED_PREFILL_SIZE=$ISL -if [ "${DP_ATTENTION}" = "true" ]; then - export SGLANG_SHARED_EXPERT_TP1=1 - export SGLANG_DP_SHARED_EXPERT_LOCAL=1 - export SGLANG_DP_USE_GATHERV=1 - export SGLANG_DP_USE_REDUCE_SCATTER=1 - export GPU_MAX_HW_QUEUES=5 - export SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE=1 - - CHUNKED_PREFILL_SIZE=$((ISL * TP)) - PARALLEL_ARGS+=( - --dp "$TP" - --enable-dp-attention - --enable-prefill-delayer - ) -fi -if [ "${EP_SIZE:-1}" -gt 1 ]; then - PARALLEL_ARGS+=(--ep-size "$EP_SIZE") - SHARED_EXPERTS_ARGS=(--disable-shared-experts-fusion) -fi - -set -x -python3 -m sglang.launch_server \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${SPEC_FLAGS[@]}" \ - --trust-remote-code \ - --disable-radix-cache \ - --attention-backend dsv4 \ - --enable-deepseek-v4-fp4-indexer \ - --cuda-graph-max-bs ${CONC} \ - --max-running-requests ${CONC} \ - --mem-fraction-static 0.90 \ - --swa-full-tokens-ratio 0.15 \ - --page-size 256 \ - --kv-cache-dtype fp8_e4m3 \ - --context-length $MAX_MODEL_LEN \ - --chunked-prefill-size $CHUNKED_PREFILL_SIZE \ - "${SHARED_EXPERTS_ARGS[@]}" \ - --tool-call-parser deepseekv4 \ - --reasoning-parser deepseek-v4 \ - --chat-template "$(dirname "$0")/../chat_templates/deepseek_v4_thinking.jinja" \ - --watchdog-timeout 1800 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# --dsv4 routes prompts through encoding_dsv4.py, emitting the -# ... framing DeepSeek-V4-Pro expects. EAGLE/MTP -# acceptance silently regresses on raw random tokens, so MTP benchmarks must -# use chat-formatted inputs (AGENTS.md). The DSv4-Pro tokenizer ships without a -# jinja chat_template, so plain --use-chat-template would crash; --dsv4 handles -# the framing directly. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --dsv4 - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_vllm.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_vllm.sh deleted file mode 100755 index f01047a7d4..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_vllm.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4-Pro on MI355X via vLLM. -# The DeepSeek-V4-Pro checkpoint is mixed-precision FP4+FP8 (FP4 MoE -# expert weights dominate the ~960 GB footprint, FP8 on attention/norm/ -# router, FP8 KV cache at runtime). InferenceX classifies this as the -# fp4 variant. -# -# Serving flags follow the validated MI355X recipe from -# vllm-project/recipes#433 (DeepSeek-V4-Pro, TP=8). DEP probes reuse the -# same ROCm recipe while switching parallelism to vLLM's DP+EP form. -# Image-pin details live in amd-master.yaml. -# -# Use the AITER MoE backend (VLLM_ROCM_USE_AITER_MOE=1 + --moe-backend aiter) -# for the FP4 MoE expert weights of deepseek-ai/DeepSeek-V4-Pro. The AITER -# MXFP4 path registers the FP4 scale parameters (w13_weight_scale / -# w2_weight_scale), so safetensors loads correctly and decode runs on the -# fused AITER experts instead of triton_unfused. -# -# --compilation-config mode=3 with FULL_AND_PIECEWISE cudagraph mode -# enables full CUDA graph capture for improved throughput on MI355X. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 -# Fuses the native-FP8 shared expert into the MXFP4 routed-expert AITER kernel. -export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -# vLLM only clamps torch threads after weight loading; cap from process start. -export OMP_NUM_THREADS=1 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -set -x -vllm serve $MODEL --port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --async-scheduling \ - --no-enable-prefix-caching \ - --distributed-executor-backend mp \ - --gpu-memory-utilization 0.8 \ - --kv-cache-dtype fp8 \ - --trust-remote-code \ - --moe-backend aiter \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_vllm_mtp.sh deleted file mode 100755 index 1e5de8ee3d..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp4_mi355x_vllm_mtp.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4-Pro on MI355X via vLLM — MTP variant of dsv4_fp4_mi355x_vllm.sh. -# Adds MTP speculative decoding per vllm-project/vllm#43385 (ROCm DeepSeek-V4 -# MTP support, merged 2026-05-24, present in v0.22.0 tagged 2026-05-29): -# --speculative-config '{"method":"mtp","num_speculative_tokens":2}'. -# -# Benchmark prompts are routed through DeepSeek-V4 chat encoding via --dsv4 -# (which auto-enables --use-chat-template). EAGLE/MTP-style spec decoding is -# trained against chat-formatted inputs; benchmarking against raw random -# prompts silently regresses the acceptance rate. -# -# All other serving flags mirror the non-MTP MI355X recipe (TP=8, -# VLLM_ROCM_USE_AITER=1, AITER MoE, FP8 KV cache, mp executor, async -# scheduling, mode=3 FULL_AND_PIECEWISE compilation). See -# dsv4_fp4_mi355x_vllm.sh for per-flag rationale. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 -# Fuses the native-FP8 shared expert into the MXFP4 routed-expert AITER kernel. -export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -# vLLM only clamps torch threads after weight loading; cap from process start. -export OMP_NUM_THREADS=1 - -SERVER_LOG=/workspace/server.log -PORT=${PORT:-8888} - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -# use 2 speculative tokens for all configs for now -NUM_SPEC_TOKENS=2 - -set -x -vllm serve $MODEL --port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --async-scheduling \ - --no-enable-prefix-caching \ - --distributed-executor-backend mp \ - --gpu-memory-utilization 0.8 \ - --kv-cache-dtype fp8 \ - --trust-remote-code \ - --moe-backend aiter \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ - --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# --dsv4 routes prompts through DeepSeek-V4 chat encoding (auto-enables -# --use-chat-template); required for meaningful MTP acceptance numbers. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --dsv4 - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_h200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_h200.sh deleted file mode 100644 index 274dee9958..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_h200.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash - -# Per https://vllm.ai/blog/deepseek-v4 the DeepSeek-V4-Pro H200 recipe uses -# the cu129 image and omits the FP4 indexer cache flag (H200 has no FP4 -# path). Max-model-len is pinned at 800k per the recipe. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -# DeepSeek-V4-Pro weights are large; engine startup can exceed the default -# 600s. Give it an hour to load. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN_ARG="--max-model-len $EVAL_MAX_MODEL_LEN" -else - MAX_MODEL_LEN_ARG="--max-model-len $MAX_MODEL_LEN" -fi - -# DP_ATTENTION=true runs DP-attention with expert parallel (DP size = TP); -# DP_ATTENTION=false runs pure tensor parallel. -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---kv-cache-dtype fp8 \ ---block-size 256 \ ---no-enable-prefix-caching \ -"${PARALLEL_ARGS[@]}" \ -"${EP_ARGS[@]}" \ -$MAX_MODEL_LEN_ARG \ ---quantization deepseek_v4_fp8 \ ---gpu-memory-utilization 0.95 \ ---max-num-seqs 512 \ ---max-num-batched-tokens 512 \ ---no-enable-flashinfer-autotune \ ---compilation-config '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}' \ ---tokenizer-mode deepseek_v4 \ ---tool-call-parser deepseek_v4 \ ---enable-auto-tool-choice \ ---reasoning-parser deepseek_v4 > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_h200_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_h200_mtp.sh deleted file mode 100755 index bf37eb2d0a..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_h200_mtp.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro H200 vLLM MTP variant of the recipe at -# https://vllm.ai/blog/deepseek-v4. Mirrors dsv4_fp8_h200.sh but adds -# --speculative-config '{"method":"mtp","num_speculative_tokens":2}' and -# routes prompts through chat-formatted encoding via --dsv4 (required for -# meaningful MTP acceptance numbers per AGENTS.md). - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -# DeepSeek-V4-Pro weights are large; engine startup can exceed the default -# 600s. Give it an hour to load. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -# Skip the cudagraph-memory estimator during the worker memory profiling -# phase — it overestimates and pushes us over the GPU memory budget on -# H200 + MTP, even though the actual cudagraph capture works fine. -export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN_ARG="--max-model-len $EVAL_MAX_MODEL_LEN" -else - MAX_MODEL_LEN_ARG="--max-model-len $MAX_MODEL_LEN" -fi - -# DP_ATTENTION=true runs DP-attention with expert parallel (DP size = TP); -# DP_ATTENTION=false runs pure tensor parallel. -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---kv-cache-dtype fp8 \ ---block-size 256 \ ---no-enable-prefix-caching \ -"${PARALLEL_ARGS[@]}" \ -"${EP_ARGS[@]}" \ -$MAX_MODEL_LEN_ARG \ ---quantization deepseek_v4_fp8 \ ---gpu-memory-utilization 0.95 \ ---max-num-seqs 512 \ ---max-num-batched-tokens 512 \ ---no-enable-flashinfer-autotune \ ---compilation-config '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}' \ ---speculative-config '{"method":"mtp","num_speculative_tokens":2}' \ ---tokenizer-mode deepseek_v4 \ ---tool-call-parser deepseek_v4 \ ---enable-auto-tool-choice \ ---reasoning-parser deepseek_v4 > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# MTP acceptance rate degrades on raw random tokens; --dsv4 routes prompts -# through chat-formatted encoding as required for speculative decoding benchmarks. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --dsv4 - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_h200_sglang.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_h200_sglang.sh deleted file mode 100644 index 3e7132ebe6..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_h200_sglang.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -SERVER_LOG="$PWD/server.log" - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -set -x -PYTHONNOUSERSITE=1 sglang serve \ - --model-path $MODEL \ - --host 0.0.0.0 \ - --port $PORT \ - --trust-remote-code \ - --tp $TP \ - --moe-runner-backend marlin \ - --chunked-prefill-size 4096 \ - --disable-flashinfer-autotune \ - --disable-radix-cache \ - --mem-fraction-static 0.88 \ - --max-running-requests "$(( CONC * 3 / 2 > 8 ? CONC * 3 / 2 : 8 ))" \ - $EVAL_CONTEXT_ARGS >> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $((CONC * 10)) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_h200_sglang_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_h200_sglang_mtp.sh deleted file mode 100644 index 788eff5b8b..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_h200_sglang_mtp.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -SERVER_LOG="$PWD/server.log" - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -set -x -PYTHONNOUSERSITE=1 sglang serve \ - --model-path $MODEL \ - --host 0.0.0.0 \ - --port $PORT \ - --trust-remote-code \ - --tp $TP \ - --moe-runner-backend marlin \ - --chunked-prefill-size 4096 \ - --disable-flashinfer-autotune \ - --disable-radix-cache \ - --mem-fraction-static 0.88 \ - --max-running-requests "$(( CONC * 3 / 2 > 8 ? CONC * 3 / 2 : 8 ))" \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - $EVAL_CONTEXT_ARGS >> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# --dsv4 routes prompts through encoding_dsv4.py (PR #1153), which emits the -# ... framing DeepSeek-V4-Pro expects. The DSv4-Pro -# tokenizer ships without a jinja chat_template, so plain --use-chat-template -# would crash; --dsv4 sidesteps that and satisfies the AGENTS.md rule that all -# MTP scripts must benchmark against chat-formatted inputs (EAGLE acceptance -# silently regresses on raw random tokens). -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $((CONC * 10)) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" \ - --dsv4 - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_mi300x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_mi300x.sh deleted file mode 100755 index b921b6054b..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_mi300x.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4-Pro FP8 single-node on MI300X (gfx942) via vLLM. -# -# EXTRAPOLATED bring-up recipe. The sglang path was abandoned: on gfx942 -# (no native FP4) the dsv4 sglang backend's nvfp4 MoE / TileLang-MLA kernels -# have no gfx942 equivalents (they exist only for gfx950/MI355X). vLLM instead -# runs the checkpoint in FP8 via --quantization deepseek_v4_fp8, which -# dequantizes the FP4 MoE experts to FP8 — the same path the H200 dsv4 vLLM -# recipe uses (H200 is also a no-FP4 SKU). Derived from: -# * same model + framework + AMD family: dsv4_fp4_mi355x_vllm.sh (ROCm vLLM -# dsv4 structure: AITER MoE, deepseek_v4 tokenizer/parser, mp executor, -# FULL_AND_PIECEWISE compile) -# * same model, FP8 path: dsv4_fp8_h200.sh (--quantization deepseek_v4_fp8) -# * same SKU, different model: minimaxm3_fp8_mi300x.sh (gfx942 vLLM/AITER) -# -# The FP4->FP8 dequant roughly doubles the MoE footprint (~1.05 TB total), -# which fits 8x192 GB only at TP8, so the sweep is TP8-only. -# -# MoE backend is left at auto (NOT --moe-backend aiter). --quantization -# deepseek_v4_fp8 only handles the dense/attention weights; the MoE experts -# stay mxfp4 and go through vLLM's mxfp4 MoE selector. On gfx942, forcing -# aiter selects AITER_MXFP4_MXFP4 (W4A4, native mxfp4) which the gfx942 kernel -# rejects ("Mxfp4 MoE backend 'AITER_MXFP4_MXFP4' does not support ... QuantKey -# (u8 ... col=32)"). With auto, vLLM's select_deepseek_v4_mxfp4_moe_backend -# takes its ROCm+DeepseekV4 branch and prefers AITER_MXFP4_BF16 (W4A16 CK, -# dequantizes weights — no native FP4), falling back to TRITON_UNFUSED. MI355X -# keeps --moe-backend aiter because gfx950 supports the W4A4 kernel. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 - -# Cap eval concurrency for gfx942's tight KV. FP8 weights (~131GB/GPU) leave -# only ~71k tokens of KV on the 192GB MI300X ("Maximum concurrency ... 7.52x" -# for a 9472-token request). The eval defaults to CONC (128) concurrent -# requests, which OOM-kills the server mid-gsm8k. Cap to the KV budget; this -# only affects run_eval (throughput jobs use CONC directly). MI325X (256GB) -# has the headroom and keeps the default. -export EVAL_CONCURRENT_REQUESTS=8 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -set -x -vllm serve $MODEL --port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --quantization deepseek_v4_fp8 \ - --async-scheduling \ - --no-enable-prefix-caching \ - --distributed-executor-backend mp \ - --gpu-memory-utilization 0.9 \ - --max-model-len "$MAX_MODEL_LEN" \ - --kv-cache-dtype fp8 \ - --trust-remote-code \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_mi300x_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_mi300x_mtp.sh deleted file mode 100755 index 0f71361f65..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_mi300x_mtp.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4-Pro FP8 single-node on MI300X (gfx942) via vLLM, MTP variant. -# -# MTP sibling of dsv4_fp8_mi300x.sh: adds --speculative-config -# '{"method":"mtp","num_speculative_tokens":2}' (DeepSeek-V4 built-in MTP) -# and --dsv4 chat-template encoding for run_benchmark_serving (EAGLE/MTP is -# trained on chat-formatted inputs). -# -# EXTRAPOLATED bring-up recipe. The sglang path was abandoned: on gfx942 -# (no native FP4) the dsv4 sglang backend's nvfp4 MoE / TileLang-MLA kernels -# have no gfx942 equivalents (they exist only for gfx950/MI355X). vLLM instead -# runs the checkpoint in FP8 via --quantization deepseek_v4_fp8, which -# dequantizes the FP4 MoE experts to FP8 — the same path the H200 dsv4 vLLM -# recipe uses (H200 is also a no-FP4 SKU). Derived from: -# * same model + framework + AMD family: dsv4_fp4_mi355x_vllm.sh (ROCm vLLM -# dsv4 structure: AITER MoE, deepseek_v4 tokenizer/parser, mp executor, -# FULL_AND_PIECEWISE compile) -# * same model, FP8 path: dsv4_fp8_h200.sh (--quantization deepseek_v4_fp8) -# * same SKU, different model: minimaxm3_fp8_mi300x.sh (gfx942 vLLM/AITER) -# -# The FP4->FP8 dequant roughly doubles the MoE footprint (~1.05 TB total), -# which fits 8x192 GB only at TP8, so the sweep is TP8-only. -# -# MoE backend is left at auto (NOT --moe-backend aiter). --quantization -# deepseek_v4_fp8 only handles the dense/attention weights; the MoE experts -# stay mxfp4 and go through vLLM's mxfp4 MoE selector. On gfx942, forcing -# aiter selects AITER_MXFP4_MXFP4 (W4A4, native mxfp4) which the gfx942 kernel -# rejects ("Mxfp4 MoE backend 'AITER_MXFP4_MXFP4' does not support ... QuantKey -# (u8 ... col=32)"). With auto, vLLM's select_deepseek_v4_mxfp4_moe_backend -# takes its ROCm+DeepseekV4 branch and prefers AITER_MXFP4_BF16 (W4A16 CK, -# dequantizes weights — no native FP4), falling back to TRITON_UNFUSED. MI355X -# keeps --moe-backend aiter because gfx950 supports the W4A4 kernel. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 - -# Cap eval concurrency for gfx942's tight KV. FP8 weights (~131GB/GPU) leave -# only ~71k tokens of KV on the 192GB MI300X ("Maximum concurrency ... 7.52x" -# for a 9472-token request). The eval defaults to CONC (128) concurrent -# requests, which OOM-kills the server mid-gsm8k. Cap to the KV budget; this -# only affects run_eval (throughput jobs use CONC directly). MI325X (256GB) -# has the headroom and keeps the default. -export EVAL_CONCURRENT_REQUESTS=8 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -# Use 2 speculative tokens (matches dsv4_fp4_mi355x_vllm_mtp.sh). -NUM_SPEC_TOKENS=2 - -set -x -vllm serve $MODEL --port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --quantization deepseek_v4_fp8 \ - --async-scheduling \ - --no-enable-prefix-caching \ - --distributed-executor-backend mp \ - --gpu-memory-utilization 0.9 \ - --max-model-len "$MAX_MODEL_LEN" \ - --kv-cache-dtype fp8 \ - --trust-remote-code \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ - --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --dsv4 - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_mi325x.sh deleted file mode 100755 index 4dafff650e..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_mi325x.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via vLLM. -# -# EXTRAPOLATED bring-up recipe. The sglang path was abandoned: on gfx942 -# (no native FP4) the dsv4 sglang backend's nvfp4 MoE / TileLang-MLA kernels -# have no gfx942 equivalents (they exist only for gfx950/MI355X). vLLM instead -# runs the checkpoint in FP8 via --quantization deepseek_v4_fp8, which -# dequantizes the FP4 MoE experts to FP8 — the same path the H200 dsv4 vLLM -# recipe uses (H200 is also a no-FP4 SKU). Derived from: -# * same model + framework + AMD family: dsv4_fp4_mi355x_vllm.sh (ROCm vLLM -# dsv4 structure: AITER MoE, deepseek_v4 tokenizer/parser, mp executor, -# FULL_AND_PIECEWISE compile) -# * same model, FP8 path: dsv4_fp8_h200.sh (--quantization deepseek_v4_fp8) -# * same SKU, different model: minimaxm3_fp8_mi325x.sh (gfx942 vLLM/AITER) -# -# The FP4->FP8 dequant roughly doubles the MoE footprint (~1.05 TB total), -# which fits 8x256 GB comfortably at TP8, so the sweep is TP8-only. -# -# MoE backend is left at auto (NOT --moe-backend aiter) — see dsv4_fp8_mi300x.sh: -# on gfx942, forcing aiter selects AITER_MXFP4_MXFP4 (W4A4 native-mxfp4) which -# the gfx942 kernel rejects; auto's ROCm+DeepseekV4 path prefers -# AITER_MXFP4_BF16 (W4A16, dequant) with a TRITON_UNFUSED fallback. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 - -# gsm8k eval at high concurrency (8k1k) OOM-kills the server: hundreds of -# concurrent 9472-token requests exceed the ~20x KV budget even on 256GB MI325X -# (c128 fit, so this was originally left at the default, but c512 crashed the -# EngineCore mid-eval). Cap the eval to a safe in-flight count; only run_eval is -# affected (throughput jobs use CONC directly). Matches the MI300X script. -export EVAL_CONCURRENT_REQUESTS=8 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -set -x -vllm serve $MODEL --port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --quantization deepseek_v4_fp8 \ - --async-scheduling \ - --no-enable-prefix-caching \ - --distributed-executor-backend mp \ - --gpu-memory-utilization 0.9 \ - --max-model-len "$MAX_MODEL_LEN" \ - --kv-cache-dtype fp8 \ - --trust-remote-code \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_mi325x_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_mi325x_mtp.sh deleted file mode 100755 index 7233da9dd1..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/dsv4_fp8_mi325x_mtp.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via vLLM, MTP variant. -# -# MTP sibling of dsv4_fp8_mi325x.sh: adds --speculative-config -# '{"method":"mtp","num_speculative_tokens":2}' (DeepSeek-V4 built-in MTP) -# and --dsv4 chat-template encoding for run_benchmark_serving. -# -# EXTRAPOLATED bring-up recipe. The sglang path was abandoned: on gfx942 -# (no native FP4) the dsv4 sglang backend's nvfp4 MoE / TileLang-MLA kernels -# have no gfx942 equivalents (they exist only for gfx950/MI355X). vLLM instead -# runs the checkpoint in FP8 via --quantization deepseek_v4_fp8, which -# dequantizes the FP4 MoE experts to FP8 — the same path the H200 dsv4 vLLM -# recipe uses (H200 is also a no-FP4 SKU). Derived from: -# * same model + framework + AMD family: dsv4_fp4_mi355x_vllm.sh (ROCm vLLM -# dsv4 structure: AITER MoE, deepseek_v4 tokenizer/parser, mp executor, -# FULL_AND_PIECEWISE compile) -# * same model, FP8 path: dsv4_fp8_h200.sh (--quantization deepseek_v4_fp8) -# * same SKU, different model: minimaxm3_fp8_mi325x.sh (gfx942 vLLM/AITER) -# -# The FP4->FP8 dequant roughly doubles the MoE footprint (~1.05 TB total), -# which fits 8x256 GB comfortably at TP8, so the sweep is TP8-only. -# -# MoE backend is left at auto (NOT --moe-backend aiter) — see dsv4_fp8_mi300x.sh: -# on gfx942, forcing aiter selects AITER_MXFP4_MXFP4 (W4A4 native-mxfp4) which -# the gfx942 kernel rejects; auto's ROCm+DeepseekV4 path prefers -# AITER_MXFP4_BF16 (W4A16, dequant) with a TRITON_UNFUSED fallback. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 - -# gsm8k eval at high concurrency (8k1k) OOM-kills the server: hundreds of -# concurrent 9472-token requests exceed the ~20x KV budget even on 256GB MI325X -# (c128 fit, so this was originally left at the default, but c512 crashed the -# EngineCore mid-eval). Cap the eval to a safe in-flight count; only run_eval is -# affected (throughput jobs use CONC directly). Matches the MI300X script. -export EVAL_CONCURRENT_REQUESTS=8 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -# Use 2 speculative tokens (matches dsv4_fp4_mi355x_vllm_mtp.sh). -NUM_SPEC_TOKENS=2 - -set -x -vllm serve $MODEL --port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --quantization deepseek_v4_fp8 \ - --async-scheduling \ - --no-enable-prefix-caching \ - --distributed-executor-backend mp \ - --gpu-memory-utilization 0.9 \ - --max-model-len "$MAX_MODEL_LEN" \ - --kv-cache-dtype fp8 \ - --trust-remote-code \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ - --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --dsv4 - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5.1_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5.1_fp4_mi355x.sh deleted file mode 100644 index 76e03b25ed..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5.1_fp4_mi355x.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash -set -x - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# ROCm / SGLang performance tuning for MI355X -export SGLANG_ROCM_FUSED_DECODE_MLA=0 -export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -export SAFETENSORS_FAST_GPU=1 - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 32)) - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -pip install -U transformers - -python3 -m sglang.launch_server \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --trust-remote-code \ - --cuda-graph-max-bs $CONC \ - --context-length $CONTEXT_LENGTH \ - --mem-fraction-static 0.85 \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ - --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 8}' \ - --nsa-prefill-backend tilelang \ - --nsa-decode-backend tilelang $EVAL_CONTEXT_ARGS \ - --kv-cache-dtype fp8_e4m3 \ - --tokenizer-worker-num $((TP*2)) \ - --enable-aiter-allreduce-fusion \ - --disable-radix-cache> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5.1_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5.1_fp4_mi355x_atom.sh deleted file mode 100644 index eaf61277c2..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5.1_fp4_mi355x_atom.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.9 - -set -x -pip install -U transformers -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --default-chat-template-kwargs '{"enable_thinking": false}' \ - --trust-remote-code \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b200.sh deleted file mode 100755 index f96acd276d..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b200.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -echo "EP_SIZE: $EP_SIZE, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---trust-remote-code \ ---tensor-parallel-size=$TP \ ---data-parallel-size 1 --expert-parallel-size $EP_SIZE \ ---disable-radix-cache \ ---quantization modelopt_fp4 \ ---kv-cache-dtype fp8_e4m3 \ ---nsa-decode-backend trtllm \ ---nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion \ ---cuda-graph-max-bs 256 \ ---max-prefill-tokens 32768 \ ---chunked-prefill-size 32768 \ ---mem-fraction-static 0.9 \ ---stream-interval 30 \ ---scheduler-recv-interval 10 \ ---tokenizer-worker-num 6 \ ---tokenizer-path $MODEL $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b200_mtp.sh deleted file mode 100755 index 4a71ac127c..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b200_mtp.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - - -export SGLANG_ENABLE_JIT_DEEPGEMM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -SERVER_LOG=/workspace/server.log - - -echo "CONC: $CONC, ISL: $ISL, OSL: $OSL" - -MEM_FRACTION_STATIC=0.85 -if [[ "$CONC" -gt 128 ]]; then - MEM_FRACTION_STATIC=0.8 -fi -echo "MEM_FRACTION_STATIC: $MEM_FRACTION_STATIC" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---trust-remote-code \ ---tensor-parallel-size=$TP \ ---data-parallel-size 1 --expert-parallel-size 1 \ ---tool-call-parser glm47 \ ---reasoning-parser glm45 \ ---kv-cache-dtype fp8_e4m3 --quantization fp8 \ ---attention-backend nsa \ ---nsa-decode-backend trtllm --nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC --max-running-requests $CONC \ ---mem-fraction-static $MEM_FRACTION_STATIC \ ---chunked-prefill-size 32768 --max-prefill-tokens 32768 \ ---enable-flashinfer-allreduce-fusion --disable-radix-cache \ ---stream-interval 30 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ ---model-loader-extra-config '{"enable_multithread_load": true}' $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b300.sh deleted file mode 100755 index 80fd4e15fe..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b300.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5 -# does not have a B300-specific recipe, so this script reuses the existing -# GLM-5 FP4 B200 SGLang recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -SERVER_LOG=/workspace/server.log - -echo "EP_SIZE: $EP_SIZE, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---tensor-parallel-size $TP \ ---data-parallel-size 1 --expert-parallel-size $EP_SIZE \ ---disable-radix-cache \ ---quantization modelopt_fp4 \ ---kv-cache-dtype fp8_e4m3 \ ---nsa-decode-backend trtllm \ ---nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion \ ---cuda-graph-max-bs 256 \ ---max-prefill-tokens 32768 \ ---chunked-prefill-size 32768 \ ---mem-fraction-static 0.9 \ ---stream-interval 30 \ ---scheduler-recv-interval 10 \ ---tokenizer-worker-num 6 \ ---tokenizer-path $MODEL_PATH $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b300_mtp.sh deleted file mode 100755 index 841381e826..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b300_mtp.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5.1 -# does not have a B300-specific recipe, so this script reuses the existing -# GLM5 FP8 B200 SGLang recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - - -export SGLANG_ENABLE_JIT_DEEPGEMM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -SERVER_LOG=/workspace/server.log - - -echo "CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---tensor-parallel-size $TP \ ---data-parallel-size 1 --expert-parallel-size 1 \ ---tool-call-parser glm47 \ ---reasoning-parser glm45 \ ---kv-cache-dtype fp8_e4m3 --quantization fp8 \ ---attention-backend nsa \ ---nsa-decode-backend trtllm --nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC --max-running-requests $CONC \ ---mem-fraction-static 0.85 \ ---chunked-prefill-size 32768 --max-prefill-tokens 32768 \ ---enable-flashinfer-allreduce-fusion --disable-radix-cache \ ---stream-interval 30 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ ---model-loader-extra-config '{"enable_multithread_load": true}' $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b200.sh deleted file mode 100755 index 7f5d3f5fc2..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b200.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export SGLANG_ENABLE_JIT_DEEPGEMM=1 - -SERVER_LOG=/workspace/server.log - - -echo "CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---trust-remote-code \ ---tensor-parallel-size=$TP \ ---data-parallel-size 1 --expert-parallel-size 1 \ ---tool-call-parser glm47 \ ---reasoning-parser glm45 \ ---kv-cache-dtype fp8_e4m3 --quantization fp8 \ ---attention-backend nsa \ ---nsa-decode-backend trtllm --nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC --max-running-requests $CONC \ ---mem-fraction-static 0.85 \ ---chunked-prefill-size 32768 --max-prefill-tokens 32768 \ ---enable-flashinfer-allreduce-fusion --disable-radix-cache \ ---stream-interval 30 \ ---model-loader-extra-config '{"enable_multithread_load": true}' $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b200_mtp.sh deleted file mode 100755 index 3a8384d0e1..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b200_mtp.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export SGLANG_ENABLE_JIT_DEEPGEMM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -SERVER_LOG=/workspace/server.log - - -echo "CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---trust-remote-code \ ---tensor-parallel-size=$TP \ ---data-parallel-size 1 --expert-parallel-size 1 \ ---tool-call-parser glm47 \ ---reasoning-parser glm45 \ ---kv-cache-dtype fp8_e4m3 --quantization fp8 \ ---attention-backend nsa \ ---nsa-decode-backend trtllm --nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC --max-running-requests $CONC \ ---mem-fraction-static 0.85 \ ---chunked-prefill-size 32768 --max-prefill-tokens 32768 \ ---enable-flashinfer-allreduce-fusion --disable-radix-cache \ ---stream-interval 30 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ ---model-loader-extra-config '{"enable_multithread_load": true}' $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b300.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b300.sh deleted file mode 100644 index 709ed2d417..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b300.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5.1 -# does not have a B300-specific recipe, so this script reuses the existing -# GLM5 FP8 B200 SGLang recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -export SGLANG_ENABLE_JIT_DEEPGEMM=1 - -SERVER_LOG=/workspace/server.log - - -echo "CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---tensor-parallel-size $TP \ ---data-parallel-size 1 --expert-parallel-size 1 \ ---tool-call-parser glm47 \ ---reasoning-parser glm45 \ ---kv-cache-dtype fp8_e4m3 --quantization fp8 \ ---attention-backend nsa \ ---nsa-decode-backend trtllm --nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC --max-running-requests $CONC \ ---mem-fraction-static 0.85 \ ---chunked-prefill-size 32768 --max-prefill-tokens 32768 \ ---enable-flashinfer-allreduce-fusion --disable-radix-cache \ ---stream-interval 30 \ ---model-loader-extra-config '{"enable_multithread_load": true}' $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b300_mtp.sh deleted file mode 100755 index 841381e826..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b300_mtp.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5.1 -# does not have a B300-specific recipe, so this script reuses the existing -# GLM5 FP8 B200 SGLang recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - - -export SGLANG_ENABLE_JIT_DEEPGEMM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -SERVER_LOG=/workspace/server.log - - -echo "CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---tensor-parallel-size $TP \ ---data-parallel-size 1 --expert-parallel-size 1 \ ---tool-call-parser glm47 \ ---reasoning-parser glm45 \ ---kv-cache-dtype fp8_e4m3 --quantization fp8 \ ---attention-backend nsa \ ---nsa-decode-backend trtllm --nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC --max-running-requests $CONC \ ---mem-fraction-static 0.85 \ ---chunked-prefill-size 32768 --max-prefill-tokens 32768 \ ---enable-flashinfer-allreduce-fusion --disable-radix-cache \ ---stream-interval 30 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ ---model-loader-extra-config '{"enable_multithread_load": true}' $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_h200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_h200.sh deleted file mode 100644 index 8dced28d48..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_h200.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -python3 -m sglang.launch_server \ - --model-path "$MODEL" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --tp-size "$TP" \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ - --mem-fraction-static 0.85 \ - --served-model-name glm-5-fp8 \ - --trust-remote-code \ - --enable-flashinfer-allreduce-fusion \ - $EVAL_CONTEXT_ARGS > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -# Server accepts glm-5-fp8 (--served-model-name); lm-eval must use that model name -if [ "${RUN_EVAL}" = "true" ]; then - export MODEL_NAME=glm-5-fp8 - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_h200_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_h200_mtp.sh deleted file mode 100755 index d2d9cc507a..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_h200_mtp.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash - -# GLM-5 FP8 on H200 (Hopper) with EAGLE / MTP speculative decoding. -# Mirrors glm5_fp8_h200.sh but adds the speculative-* flags. We keep the -# server-arg shape from the non-MTP H200 recipe (sglang defaults — no -# nsa/trtllm-mha) since those backends are Blackwell-specific and not -# applicable to Hopper. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor - -set -x -python3 -m sglang.launch_server \ - --model-path "$MODEL" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --tp-size "$TP" \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ - --mem-fraction-static 0.85 \ - --served-model-name glm-5-fp8 \ - --trust-remote-code \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - $EVAL_CONTEXT_ARGS > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - export MODEL_NAME=glm-5-fp8 - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi325x.sh deleted file mode 100755 index 6b2d48d23f..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi325x.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi - -start_gpu_monitor - -# Launch args follow sglang issue #25672 comment 4485916205: -# tilelang NSA backends + fp8_e4m3 KV cache + multithread model load. -python3 -m sglang.launch_server \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --data-parallel-size 1 \ - --trust-remote-code \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ - --tokenizer-worker-num 6 \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.80 \ - --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 8}' \ - --nsa-prefill-backend tilelang \ - --nsa-decode-backend tilelang \ - --kv-cache-dtype fp8_e4m3 \ - $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi325x_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi325x_mtp.sh deleted file mode 100755 index 8beaedf946..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi325x_mtp.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash - -# GLM-5 FP8 on MI325X with EAGLE / MTP speculative decoding. -# Mirrors glm5_fp8_mi325x.sh and adds the speculative-* flags. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi - -start_gpu_monitor - -# Launch args follow sglang issue #25672 comment 4485916205: -# tilelang NSA backends + fp8_e4m3 KV cache + multithread model load, -# plus EAGLE/MTP speculative decoding. -python3 -m sglang.launch_server \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --ep-size $EP_SIZE \ - --trust-remote-code \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ - --tokenizer-worker-num 6 \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.80 \ - --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 8}' \ - --nsa-prefill-backend tilelang \ - --nsa-decode-backend tilelang \ - --kv-cache-dtype fp8_e4m3 \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x.sh deleted file mode 100755 index f130cf9c9a..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# ROCm / SGLang performance tuning for MI355X -export SGLANG_ROCM_FUSED_DECODE_MLA=0 -export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -export SAFETENSORS_FAST_GPU=1 - -SERVER_LOG=/workspace/server.log - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -python3 -m sglang.launch_server \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --trust-remote-code \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ - --mem-fraction-static 0.85 \ - --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 8}' \ - --nsa-prefill-backend tilelang \ - --nsa-decode-backend tilelang $EVAL_CONTEXT_ARGS \ - --kv-cache-dtype fp8_e4m3 \ - --max-running-requests ${CONC} \ - --cuda-graph-max-bs ${CONC} \ - --disable-radix-cache> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x_atom.sh deleted file mode 100644 index eaf61277c2..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x_atom.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.9 - -set -x -pip install -U transformers -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --default-chat-template-kwargs '{"enable_thinking": false}' \ - --trust-remote-code \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x_mtp.sh deleted file mode 100755 index 80a8488e9f..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x_mtp.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash -set -x - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# ROCm / SGLang performance tuning for MI355X -export SGLANG_ROCM_FUSED_DECODE_MLA=0 -export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -export SAFETENSORS_FAST_GPU=1 -export SGLANG_ENABLE_SPEC_V2=1 - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 32)) - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -python3 -m sglang.launch_server \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --trust-remote-code \ - --cuda-graph-max-bs $CONC \ - --context-length $CONTEXT_LENGTH \ - --mem-fraction-static 0.85 \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ - --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 8}' \ - --nsa-prefill-backend tilelang \ - --nsa-decode-backend tilelang $EVAL_CONTEXT_ARGS \ - --kv-cache-dtype fp8_e4m3 \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --tokenizer-worker-num $((TP*2)) \ - --disable-radix-cache> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_b200.sh deleted file mode 100644 index 4a41811bfd..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_b200.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN=$((ISL + OSL + 20)) -elif [ "$ISL" = "8192" ] || [ "$OSL" = "8192" ]; then - CALCULATED_MAX_MODEL_LEN=$((ISL + OSL + 256)) -else - CALCULATED_MAX_MODEL_LEN=${MAX_MODEL_LEN:-10240} -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CALCULATED_MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -cat > config.yaml << EOF -kv-cache-dtype: fp8 -compilation-config: '{"pass_config":{"fuse_allreduce_rms":true,"eliminate_noops":true}}' -no-enable-prefix-caching: true -max-cudagraph-capture-size: 2048 -max-num-batched-tokens: 8192 -max-model-len: $CALCULATED_MAX_MODEL_LEN -EOF - -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8=1 - -SERVER_LOG=/workspace/server.log - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---config config.yaml \ ---gpu-memory-utilization 0.9 \ ---tensor-parallel-size $TP \ ---max-num-seqs 512 > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_b200_trt.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_b200_trt.sh deleted file mode 100644 index 2871774cbe..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_b200_trt.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env bash - -# Source benchmark utilities early -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -# GPTOSS TRTLLM Deployment Guide: -# https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.md - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi -SERVER_LOG=/workspace/server.log - -# ========= Determine DP_ATTENTION, EP_SIZE and MOE_BACKEND based on ISL, OSL, CONC ========= -MOE_BACKEND="TRTLLM" - -echo "MOE_BACKEND set to '$MOE_BACKEND'" - -EXTRA_CONFIG_FILE="gptoss-fp4.yml" -export TRTLLM_ENABLE_PDL=1 - -cat > $EXTRA_CONFIG_FILE << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $CONC -enable_attention_dp: $DP_ATTENTION -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: $MOE_BACKEND -EOF - -if [[ "$DP_ATTENTION" == "true" ]]; then - # DISABLE All2All for MoE TP - if [[ "$EP_SIZE" -eq 1 ]]; then - # DTP Alltoall Environment variables for EP_SIZE == 1 - export TRTLLM_FORCE_ALLTOALL_METHOD="NotEnabled" - elif [[ "$EP_SIZE" -gt 1 ]]; then - # DEP - export TRTLLM_MOE_ALLTOALL_BACKEND="mnnvlthroughput" - export TRTLLM_FORCE_ALLTOALL_METHOD="MNNVL" - export TRTLLM_MOE_A2A_WORKSPACE_MB="2048" - fi - cat << EOF >> $EXTRA_CONFIG_FILE -attention_dp_config: - enable_balance: true -EOF -fi - -echo "Generated config file contents:" -cat $EXTRA_CONFIG_FILE - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -MAX_NUM_TOKENS=20000 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -# Launch TRT-LLM server -mpirun -n 1 --oversubscribe --allow-run-as-root \ - trtllm-serve $MODEL --port=$PORT \ - --trust_remote_code \ - --backend=pytorch \ - --max_batch_size 512 \ - --max_seq_len=$MAX_MODEL_LEN \ - --max_num_tokens=$MAX_NUM_TOKENS \ - --tp_size=$TP --ep_size=$EP_SIZE \ - --extra_llm_api_options=$EXTRA_CONFIG_FILE \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h100.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h100.sh deleted file mode 100644 index 8b373b4468..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h100.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -MAX_MODEL_LEN=10240 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -cat > config.yaml << EOF -no-enable-prefix-caching: true -max-cudagraph-capture-size: 2048 -max-num-batched-tokens: 8192 -max-model-len: $MAX_MODEL_LEN -EOF - -export PYTHONNOUSERSITE=1 -export VLLM_MXFP4_USE_MARLIN=1 -SERVER_LOG=/workspace/server.log - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --host=0.0.0.0 --port=$PORT \ ---config config.yaml \ ---gpu-memory-utilization=0.9 \ ---tensor-parallel-size=$TP \ ---max-num-seqs=$CONC > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h200.sh deleted file mode 100644 index ffe225f0b6..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h200.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -pip install datasets pandas - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN=$((ISL + OSL + 20)) -elif [ "$ISL" = "8192" ] || [ "$OSL" = "8192" ]; then - CALCULATED_MAX_MODEL_LEN=$((ISL + OSL + 256)) -else - CALCULATED_MAX_MODEL_LEN=${MAX_MODEL_LEN:-10240} -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CALCULATED_MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -# Create config.yaml -cat > config.yaml << EOF -no-enable-prefix-caching: true -max-cudagraph-capture-size: 2048 -max-num-batched-tokens: 8192 -max-model-len: $CALCULATED_MAX_MODEL_LEN -EOF - -SERVER_LOG=/workspace/server.log -export TORCH_CUDA_ARCH_LIST="9.0" - -export VLLM_MXFP4_USE_MARLIN=1 - -PYTHONNOUSERSITE=1 vllm serve $MODEL --host 0.0.0.0 --port $PORT \ - --config config.yaml \ - --gpu-memory-utilization 0.9 \ - --tensor-parallel-size $TP \ - --max-num-seqs $CONC > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h200_trt.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h200_trt.sh deleted file mode 100644 index 84eaa67370..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h200_trt.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi -SERVER_LOG=/workspace/server.log - -set +x - -export TRTLLM_ENABLE_PDL=1 - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -cat > gptoss-config.yml << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $CONC -enable_attention_dp: $DP_ATTENTION -kv_cache_config: - dtype: auto - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -EOF - -MAX_NUM_TOKENS=20000 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -PYTHONNOUSERSITE=1 mpirun -n 1 --oversubscribe --allow-run-as-root \ -trtllm-serve $MODEL \ ---max_batch_size $CONC \ ---max_num_tokens $MAX_NUM_TOKENS \ ---max_seq_len=$MAX_MODEL_LEN \ ---backend pytorch \ ---extra_llm_api_options gptoss-config.yml \ ---ep_size=$EP_SIZE \ ---trust_remote_code \ ---gpus_per_node 8 \ ---host 0.0.0.0 \ ---port $PORT \ ---tp_size=$TP \ ---pp_size=1 \ -> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi300x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi300x.sh deleted file mode 100644 index 833718b5ad..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi300x.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# If the machine runs a MEC FW older than 177, RCCL -# cannot reclaim some memory. -# Disable that features to avoid crashes. -# This is related to the changes in the driver at: -# https://rocm.docs.amd.com/en/docs-6.4.3/about/release-notes.html#amdgpu-driver-updates -version=`rocm-smi --showfw | grep MEC | head -n 1 | awk '{print $NF}'` -if [[ "$version" == "" || $version -lt 177 ]]; then - export HSA_NO_SCRATCH_RECLAIM=1 -fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export AMDGCN_USE_BUFFER_OPS=0 -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -ATTN_BACKEND="--attention-backend ROCM_AITER_UNIFIED_ATTN" -FUSE_ROPE_KVCACHE="-cc.pass_config.fuse_rope_kvcache=True -cc.use_inductor_graph_partition=True" - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ - $ATTN_BACKEND $FUSE_ROPE_KVCACHE \ - --tensor-parallel-size=$TP \ - --gpu-memory-utilization 0.95 \ - --max-model-len $MAX_MODEL_LEN \ - --block-size=64 \ - --no-enable-prefix-caching > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi325x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi325x.sh deleted file mode 100644 index 833718b5ad..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi325x.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# If the machine runs a MEC FW older than 177, RCCL -# cannot reclaim some memory. -# Disable that features to avoid crashes. -# This is related to the changes in the driver at: -# https://rocm.docs.amd.com/en/docs-6.4.3/about/release-notes.html#amdgpu-driver-updates -version=`rocm-smi --showfw | grep MEC | head -n 1 | awk '{print $NF}'` -if [[ "$version" == "" || $version -lt 177 ]]; then - export HSA_NO_SCRATCH_RECLAIM=1 -fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export AMDGCN_USE_BUFFER_OPS=0 -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -ATTN_BACKEND="--attention-backend ROCM_AITER_UNIFIED_ATTN" -FUSE_ROPE_KVCACHE="-cc.pass_config.fuse_rope_kvcache=True -cc.use_inductor_graph_partition=True" - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ - $ATTN_BACKEND $FUSE_ROPE_KVCACHE \ - --tensor-parallel-size=$TP \ - --gpu-memory-utilization 0.95 \ - --max-model-len $MAX_MODEL_LEN \ - --block-size=64 \ - --no-enable-prefix-caching > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi355x.sh deleted file mode 100644 index b4a4d03732..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi355x.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# If the machine runs a MEC FW older than 177, RCCL -# cannot reclaim some memory. -# Disable that features to avoid crashes. -# This is related to the changes in the driver at: -# https://rocm.docs.amd.com/en/docs-6.4.3/about/release-notes.html#amdgpu-driver-updates -version=`rocm-smi --showfw | grep MEC | head -n 1 | awk '{print $NF}'` -if [[ "$version" == "" || $version -lt 177 ]]; then - export HSA_NO_SCRATCH_RECLAIM=1 -fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export AMDGCN_USE_BUFFER_OPS=0 -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_TRITON_ROPE=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -ATTN_BACKEND="--attention-backend ROCM_AITER_UNIFIED_ATTN" -FUSE_ROPE_KVCACHE="-cc.pass_config.fuse_rope_kvcache=True -cc.use_inductor_graph_partition=True" - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ - $ATTN_BACKEND $FUSE_ROPE_KVCACHE \ - --tensor-parallel-size=$TP \ - --gpu-memory-utilization 0.95 \ - --max-model-len $MAX_MODEL_LEN \ - --block-size=64 \ - --no-enable-prefix-caching > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi355x_atom.sh deleted file mode 100644 index e60c389d3b..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi355x_atom.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CALCULATED_MAX_MODEL_LEN=" --max-model-len $EVAL_MAX_MODEL_LEN " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.9 - -set -x - -BLOCK_SIZE=${BLOCK_SIZE:-16} -export ATOM_GPT_OSS_MODEL=1 #TODO remove this -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --block-size $BLOCK_SIZE > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_fp4_b200.sh deleted file mode 100644 index 793bf2553c..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_fp4_b200.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -GMU=0.90 -PREFILL_SCHEDULE_ARGS=() -if [ "${DP_ATTENTION:-false}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") - GMU=0.80 - PREFILL_SCHEDULE_ARGS=(--prefill-schedule-interval 4) -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export VLLM_USE_V2_MODEL_RUNNER=0 -export VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS="" - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -# vLLM v0.20.2+'s CUDA-graph memory profiler pre-reserves ~57 GB/GPU upfront -# (~32% of total), which collides with --gpu-memory-utilization=0.90 and -# leaves negative space for the KV cache. Disable the profiler — our 0.90 -# already leaves ~18 GB/GPU as safety net (same pattern as -# benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b200.sh). -export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 - -set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ -"${PARALLEL_ARGS[@]}" \ -"${EP_ARGS[@]}" \ -"${PREFILL_SCHEDULE_ARGS[@]}" \ ---gpu-memory-utilization "$GMU" \ ---max-model-len $MAX_MODEL_LEN \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---kv-cache-dtype fp8 \ ---max-cudagraph-capture-size "$((CONC * 2))" \ ---stream-interval 32 \ ---attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}' \ ---linear-backend flashinfer_cutlass \ ---no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_fp4_b300.sh deleted file mode 100755 index 8f1906aaa3..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_fp4_b300.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html -# does not have a B300-specific recipe, so this script reuses the existing -# Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -GMU=0.90 -PREFILL_SCHEDULE_ARGS=() -if [ "${DP_ATTENTION:-false}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") - GMU=0.85 - PREFILL_SCHEDULE_ARGS=(--prefill-schedule-interval 4) -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - - -nvidia-smi - -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export VLLM_USE_V2_MODEL_RUNNER=0 -export VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS="" -export VLLM_RPC_TIMEOUT=600000 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ -"${PARALLEL_ARGS[@]}" \ -"${EP_ARGS[@]}" \ -"${PREFILL_SCHEDULE_ARGS[@]}" \ ---gpu-memory-utilization "$GMU" \ ---max-model-len $MAX_MODEL_LEN \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---kv-cache-dtype fp8 \ ---max-cudagraph-capture-size "$((CONC * 2))" \ ---stream-interval 32 \ ---attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}' \ ---linear-backend flashinfer_cutlass \ ---no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_fp4_mi355x.sh deleted file mode 100755 index ecd3d0e67a..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_fp4_mi355x.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Install amd-quark for MXFP4 quantization support -# need to manually install due to ROCm vLLM bug -# https://github.com/vllm-project/vllm/issues/35633 -pip install amd-quark - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -if [ "$MAX_MODEL_LEN" -lt 9472 ]; then - MAX_MODEL_LEN=9472 -fi - -# If the machine runs a MEC FW older than 177, RCCL -# cannot reclaim some memory. -# Disable that features to avoid crashes. -# This is related to the changes in the driver at: -# https://rocm.docs.amd.com/en/docs-6.4.3/about/release-notes.html#amdgpu-driver-updates -version=`rocm-smi --showfw | grep MEC | head -n 1 | awk '{print $NF}'` -if [[ "$version" == "" || $version -lt 177 ]]; then - export HSA_NO_SCRATCH_RECLAIM=1 -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 -export VLLM_ROCM_USE_SKINNY_GEMM=0 -export AITER_MXFP4_INTERMEDIATE=1 -export AITER_BYPASS_TUNE_CONFIG=0 -export AITER_MOE_SORT_BACKEND=auto -export OMP_NUM_THREADS=1 - -# Disable AITER RMSNorm for TP < 8 due to accuracy issues -if [ "${TP}" -lt 8 ]; then - export VLLM_ROCM_USE_AITER_RMSNORM=0 -fi - -if [ "${EP_SIZE:-0}" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ -$EP \ ---gpu-memory-utilization 0.85 \ ---max-model-len $MAX_MODEL_LEN \ ---kv-cache-dtype fp8 \ ---block-size 16 \ ---max-num-batched-tokens 16384 \ ---max-num-seqs 512 \ ---async-scheduling \ ---trust-remote-code \ ---no-enable-prefix-caching \ ---mm-encoder-tp-mode data > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_fp4_mi355x_atom.sh deleted file mode 100755 index 38d625d1d6..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_fp4_mi355x_atom.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - - -export ATOM_DISABLE_MMAP=true # Model load faster. -export AITER_QUICK_REDUCE_QUANTIZATION=INT4 -export AITER_MXFP4_INTERMEDIATE=1 -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --scheduler-delay-factor 1 \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --trust-remote-code \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_b200.sh deleted file mode 100755 index cbef22d672..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_b200.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -export PYTHONNOUSERSITE=1 -export VLLM_USE_FLASHINFER_MOE_INT4=1 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---gpu-memory-utilization 0.95 \ ---tensor-parallel-size $TP \ ---max-model-len $MAX_MODEL_LEN \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---trust-remote-code \ ---no-enable-prefix-caching > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_b300.sh b/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_b300.sh deleted file mode 100755 index e661401410..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_b300.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html -# does not have a B300-specific recipe, so this script reuses the existing -# Kimi-K2.5 INT4 B200 vLLM recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - - -nvidia-smi - -export PYTHONNOUSERSITE=1 -export VLLM_USE_FLASHINFER_MOE_INT4=1 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---gpu-memory-utilization 0.95 \ ---tensor-parallel-size $TP \ ---max-model-len $MAX_MODEL_LEN \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---trust-remote-code \ ---no-enable-prefix-caching > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_h200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_h200.sh deleted file mode 100755 index 1f18032ff5..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_h200.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -export PYTHONNOUSERSITE=1 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -# following https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html recipe - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---gpu-memory-utilization 0.95 \ ---tensor-parallel-size $TP \ ---max-model-len $MAX_MODEL_LEN \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---trust-remote-code \ ---no-enable-prefix-caching > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_mi300x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_mi300x.sh deleted file mode 100755 index bb5145a667..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_mi300x.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log - -# following AMD andy luo's recipe -# https://x.com/linluo77/status/2017024513595301985 -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -export VLLM_ROCM_USE_AITER=1 -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size=64 \ ---trust-remote-code \ ---no-enable-prefix-caching \ ---max-num-seqs 256 \ ---mm-encoder-tp-mode data > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_mi325x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_mi325x.sh deleted file mode 100755 index bb5145a667..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_mi325x.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log - -# following AMD andy luo's recipe -# https://x.com/linluo77/status/2017024513595301985 -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -export VLLM_ROCM_USE_AITER=1 -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size=64 \ ---trust-remote-code \ ---no-enable-prefix-caching \ ---max-num-seqs 256 \ ---mm-encoder-tp-mode data > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_mi355x.sh deleted file mode 100755 index dc16f1e532..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/kimik2.5_int4_mi355x.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -export VLLM_ROCM_USE_AITER=1 -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size=64 \ ---trust-remote-code \ ---no-enable-prefix-caching \ ---max-num-seqs 256 \ ---moe-backend flydsl \ ---compilation-config '{"pass_config": {"fuse_allreduce_rms": false}}' \ ---mm-encoder-tp-mode data > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b200.sh deleted file mode 100755 index fc7877a1ca..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b200.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---kv-cache-dtype fp8 \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b200_trt.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b200_trt.sh deleted file mode 100755 index 2fd9d79974..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b200_trt.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env bash - -# Source benchmark utilities early -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -MAX_NUM_TOKENS=16384 -MAX_CAPTURE_TOKENS=$(( MAX_NUM_TOKENS < CONC * ISL ? MAX_NUM_TOKENS : CONC * ISL )) -CAPTURE_TOKENS_LIST=(1 2 4 8 12 16 24 32 48 64 96 128 192 256 384 512 768) -CAPTURE_TOKENS_LIST+=( $(seq 1024 128 2047)) -CAPTURE_TOKENS_LIST+=( $(seq 2048 256 4095)) -if [[ $MAX_CAPTURE_TOKENS -ge 4096 ]]; then - CAPTURE_TOKENS_LIST+=( $(seq 4096 512 $MAX_CAPTURE_TOKENS)) -fi -CAPTURE_TOKENS_LIST=$(printf "%s, " "${CAPTURE_TOKENS_LIST[@]}") - -CAPTURE_BATCH_LIST=(1 2 4 8 12 ) -if [[ $CONC -ge 16 ]]; then - MAX_CAPTURE_BATCH=$(( CONC < 256 ? CONC : 255 )) - CAPTURE_BATCH_LIST+=( $(seq 16 8 $MAX_CAPTURE_BATCH )) -fi -if [[ $CONC -ge 256 ]]; then - MAX_CAPTURE_BATCH=$(( CONC < 512 ? CONC : 511 )) - CAPTURE_BATCH_LIST+=( $(seq 256 16 $MAX_CAPTURE_BATCH)) -fi -if [[ $CONC -ge 512 ]]; then - MAX_CAPTURE_BATCH=$(( CONC < 768 ? CONC : 767 )) - CAPTURE_BATCH_LIST+=( $(seq 512 32 $MAX_CAPTURE_BATCH)) -fi -if [[ $CONC -ge 1024 ]]; then - CAPTURE_BATCH_LIST+=( $(seq 768 64 $CONC)) -fi -CAPTURE_BATCH_LIST=$(printf "%s, " "${CAPTURE_BATCH_LIST[@]}") -MAX_CAPTURE_TOKENS=$(( CONC < 16 ? 4096 : MAX_NUM_TOKENS )) - -CONFIG_FILE="minimax-fp4.yaml" -cat << EOF > $CONFIG_FILE -cuda_graph_config: - enable_padding: true - batch_sizes: [${CAPTURE_BATCH_LIST%, }] -moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true -enable_attention_dp: $DP_ATTENTION -torch_compile_config: - capture_num_tokens: [${CAPTURE_TOKENS_LIST%, }] - enable_piecewise_cuda_graph: true -stream_interval: 100 -print_iter_log: true -max_num_tokens: $MAX_NUM_TOKENS -kv_cache_config: - free_gpu_memory_fraction: 0.9 - enable_block_reuse: False - dtype: fp8 -scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED -nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -max_seq_len: $MAX_MODEL_LEN -num_postprocess_workers: 4 -EOF - -if [[ $DP_ATTENTION == true ]]; then -cat << EOF >> $CONFIG_FILE -attention_dp_config: - enable_balance: true -EOF -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi -SERVER_LOG=/workspace/server.log -PORT=${PORT:-8888} - -echo "Generated config file contents:" -cat $CONFIG_FILE - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -# Launch TRT-LLM server -mpirun -n 1 --oversubscribe --allow-run-as-root \ - trtllm-serve $MODEL --port=$PORT \ - --trust_remote_code \ - --backend=pytorch \ - --max_batch_size $CONC \ - --tp_size=$TP --ep_size=$EP_SIZE \ - --config=$CONFIG_FILE \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b300.sh deleted file mode 100755 index 33492aada8..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b300.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/MiniMax/MiniMax-M2.html -# does not have a B300-specific recipe, so this script reuses the existing -# MiniMax-M2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -SERVER_LOG=/workspace/server.log - -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size 1 --data-parallel-size $TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size $TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size $TP" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL_PATH --served-model-name $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---kv-cache-dtype fp8 \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b300_trt.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b300_trt.sh deleted file mode 100755 index a4961b0c17..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b300_trt.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env bash - -# Source benchmark utilities early -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -MAX_NUM_TOKENS=16384 -MAX_CAPTURE_TOKENS=$(( MAX_NUM_TOKENS < CONC * ISL ? MAX_NUM_TOKENS : CONC * ISL )) -CAPTURE_TOKENS_LIST=(1 2 4 8 12 16 24 32 48 64 96 128 192 256 384 512 768) -CAPTURE_TOKENS_LIST+=( $(seq 1024 128 2047)) -CAPTURE_TOKENS_LIST+=( $(seq 2048 256 4095)) -if [[ $MAX_CAPTURE_TOKENS -ge 4096 ]]; then - CAPTURE_TOKENS_LIST+=( $(seq 4096 512 $MAX_CAPTURE_TOKENS)) -fi -CAPTURE_TOKENS_LIST=$(printf "%s, " "${CAPTURE_TOKENS_LIST[@]}") - -CAPTURE_BATCH_LIST=(1 2 4 8 12 ) -if [[ $CONC -ge 16 ]]; then - MAX_CAPTURE_BATCH=$(( CONC < 256 ? CONC : 255 )) - CAPTURE_BATCH_LIST+=( $(seq 16 8 $MAX_CAPTURE_BATCH )) -fi -if [[ $CONC -ge 256 ]]; then - MAX_CAPTURE_BATCH=$(( CONC < 512 ? CONC : 511 )) - CAPTURE_BATCH_LIST+=( $(seq 256 16 $MAX_CAPTURE_BATCH)) -fi -if [[ $CONC -ge 512 ]]; then - MAX_CAPTURE_BATCH=$(( CONC < 768 ? CONC : 767 )) - CAPTURE_BATCH_LIST+=( $(seq 512 32 $MAX_CAPTURE_BATCH)) -fi -if [[ $CONC -ge 1024 ]]; then - CAPTURE_BATCH_LIST+=( $(seq 768 64 $CONC)) -fi -CAPTURE_BATCH_LIST=$(printf "%s, " "${CAPTURE_BATCH_LIST[@]}") -MAX_CAPTURE_TOKENS=$(( CONC < 16 ? 4096 : MAX_NUM_TOKENS )) - -CONFIG_FILE="minimax-fp4.yaml" -cat << EOF > $CONFIG_FILE -cuda_graph_config: - enable_padding: true - batch_sizes: [${CAPTURE_BATCH_LIST%, }] -moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true -enable_attention_dp: $DP_ATTENTION -torch_compile_config: - capture_num_tokens: [${CAPTURE_TOKENS_LIST%, }] - enable_piecewise_cuda_graph: true -stream_interval: 100 -print_iter_log: true -max_num_tokens: $MAX_NUM_TOKENS -kv_cache_config: - free_gpu_memory_fraction: 0.9 - enable_block_reuse: False - dtype: fp8 -scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED -nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -max_seq_len: $MAX_MODEL_LEN -num_postprocess_workers: 4 -EOF - -if [[ $DP_ATTENTION == true ]]; then -cat << EOF >> $CONFIG_FILE -attention_dp_config: - enable_balance: true -EOF -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE. -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -SERVER_LOG=/workspace/server.log -PORT=${PORT:-8888} - -echo "Generated config file contents:" -cat $CONFIG_FILE - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -# Launch TRT-LLM server -mpirun -n 1 --oversubscribe --allow-run-as-root \ - trtllm-serve "$MODEL_PATH" --port=$PORT \ - --trust_remote_code \ - --backend=pytorch \ - --max_batch_size $CONC \ - --tp_size=$TP --ep_size=$EP_SIZE \ - --config=$CONFIG_FILE \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_mi355x.sh deleted file mode 100755 index 806c59278f..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_mi355x.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_USE_RUST_FRONTEND=1 -EXTRA_VLLM_ARGS="" -# if [ "$TP" -ge 4 ]; then -# # AITER CK fused MoE kernels lack compiled tiles for N=intermediate_size/TP -# # when TP>=4 (TP=4, N=384). Disable AITER MoE to fall back to triton, but keep -# # AITER attention. See: https://github.com/vllm-project/vllm/issues/35637 -# export VLLM_ROCM_USE_AITER_MOE=0 -# EXTRA_VLLM_ARGS="--attention-backend ROCM_AITER_UNIFIED_ATTN" -# pip install amd-quark 2>/dev/null || true -# fi - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ -$EP \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---kv-cache-dtype fp8 \ ---block-size=32 \ ---no-enable-prefix-caching \ ---attention-backend "ROCM_AITER_FA" \ ---trust-remote-code \ -$EXTRA_VLLM_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_mi355x_atom.sh deleted file mode 100644 index 6730aded2b..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_mi355x_atom.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --trust-remote-code \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_b200.sh deleted file mode 100755 index 9897afca35..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_b200.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ -$EP \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size=32 \ ---kv-cache-dtype fp8 \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_b300.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_b300.sh deleted file mode 100755 index 14e853ce97..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_b300.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/MiniMax/MiniMax-M2.html -# does not have a B300-specific recipe, so this script reuses the existing -# MiniMax-M2.5 FP8 B200 vLLM recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -SERVER_LOG=/workspace/server.log - -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL_PATH --served-model-name $MODEL --port $PORT \ ---tensor-parallel-size $TP \ -$EP \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 32 \ ---kv-cache-dtype fp8 \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_h100.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_h100.sh deleted file mode 100755 index 012c8b5355..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_h100.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -export PYTHONNOUSERSITE=1 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---tensor-parallel-size=$TP \ -$EP \ ---trust-remote-code \ ---enable-auto-tool-choice \ ---tool-call-parser minimax_m2 \ ---reasoning-parser minimax_m2_append_think \ ---compilation-config '{"mode":3,"pass_config":{"fuse_minimax_qk_norm":true}}' \ ---gpu-memory-utilization 0.9 \ -> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_h200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_h200.sh deleted file mode 100755 index bb53e0a06f..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_h200.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -export PYTHONNOUSERSITE=1 -export SAFETENSORS_FAST_GPU=1 -export VLLM_USE_DEEP_GEMM=0 -export VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER=0 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -COMPILATION_CONFIG=${COMPILATION_CONFIG:-'{"mode":3,"cudagraph_mode":"PIECEWISE","pass_config":{"fuse_minimax_qk_norm":true}}'} -MAX_NUM_SEQS=${MAX_NUM_SEQS:-512} -MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-32768} - -if [ "$EP_SIZE" -gt 1 ]; then - EP=(--enable-expert-parallel) -else - EP=() -fi - -if [ "$ISL" = "8192" ]; then - ATTN_BACKEND="FLASH_ATTN" - AUTOTUNE_FLAG=() -else - ATTN_BACKEND="FLASHINFER" - AUTOTUNE_FLAG=(--enable-flashinfer-autotune) -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ ---tensor-parallel-size="$TP" \ -"${EP[@]}" \ ---gpu-memory-utilization 0.95 \ ---max-model-len "$MAX_MODEL_LEN" \ ---max-num-seqs "$MAX_NUM_SEQS" \ ---max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" \ ---kv-cache-dtype fp8 \ ---moe-backend triton \ ---attention-backend "$ATTN_BACKEND" \ -"${AUTOTUNE_FLAG[@]}" \ ---compilation-config "$COMPILATION_CONFIG" \ ---no-enable-prefix-caching \ ---trust-remote-code > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi300x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi300x.sh deleted file mode 100755 index 8a95dc138c..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi300x.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size=32 \ ---no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi325x.sh deleted file mode 100755 index 3b74e845e2..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi325x.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 - -ENABLE_SHUFFLE_KV_CACHE_LAYOUT=0 -if [[ "$TP" == "2" && "$EP_SIZE" == "1" ]] && (( CONC <= 16 )); then - ENABLE_SHUFFLE_KV_CACHE_LAYOUT=1 -elif [[ "$TP" == "8" && "$EP_SIZE" == "8" ]] && (( CONC <= 64 )); then - ENABLE_SHUFFLE_KV_CACHE_LAYOUT=1 -fi -if (( ENABLE_SHUFFLE_KV_CACHE_LAYOUT )); then - export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 -fi - -SERVER_LOG=/workspace/server.log - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ -$EP \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size=32 \ ---no-enable-prefix-caching \ ---attention-backend ROCM_AITER_FA \ ---compilation-config '{"mode":3,"cudagraph_mode":"PIECEWISE"}' \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi355x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi355x.sh deleted file mode 100755 index 5093a56d64..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi355x.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=0 -VLLM_BLOCK_SIZE=32 -ASYNC_SCHEDULING_ARGS="" - -if [[ "$ISL" == "1024" && "$OSL" == "1024" ]]; then - if [[ "$TP" == "8" && "$EP_SIZE" == "8" ]]; then - ASYNC_SCHEDULING_ARGS="--no-async-scheduling" - echo "1k1k TP8/EP8: using block size 32, shuffle disabled, async scheduling disabled." - elif (( CONC <= 128 )); then - export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 - VLLM_BLOCK_SIZE=16 - ASYNC_SCHEDULING_ARGS="--no-async-scheduling" - echo "1k1k c${CONC}: using block size 16, shuffle enabled, async scheduling disabled." - else - export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 - VLLM_BLOCK_SIZE=16 - echo "1k1k c${CONC}: using block size 16, shuffle enabled, async scheduling enabled." - fi -elif [[ "$ISL" == "8192" && "$OSL" == "1024" ]]; then - if [[ "$TP" == "8" && "$EP_SIZE" == "8" ]]; then - export VLLM_ROCM_USE_AITER_MOE=0 - ASYNC_SCHEDULING_ARGS="--no-async-scheduling" - echo "8k1k TP8/EP8: using block size 32, shuffle disabled, AITER MoE disabled, async scheduling disabled." - elif (( CONC < 64 )); then - ASYNC_SCHEDULING_ARGS="--no-async-scheduling" - echo "8k1k c${CONC}: using block size 32, shuffle disabled, async scheduling disabled." - elif (( CONC == 64 )); then - ASYNC_SCHEDULING_ARGS="--no-async-scheduling" - export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 - VLLM_BLOCK_SIZE=16 - echo "8k1k c64: using block size 16, shuffle enabled, async scheduling disabled." - else - export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 - VLLM_BLOCK_SIZE=16 - echo "8k1k c${CONC}: using block size 16, shuffle enabled, async scheduling enabled." - fi -fi - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ -$EP \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---kv-cache-dtype fp8 \ ---block-size=$VLLM_BLOCK_SIZE \ ---no-enable-prefix-caching \ ---attention-backend "ROCM_AITER_FA" \ -$ASYNC_SCHEDULING_ARGS \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi355x_atom.sh deleted file mode 100755 index 325c977262..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi355x_atom.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.9 - -set -x - -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --trust-remote-code \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_b200.sh deleted file mode 100755 index 8b4dd8ff03..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_b200.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 NVFP4 B200 single-node vLLM recipe. -# Same shape as minimaxm3_fp8_b200.sh but uses the nvidia/MiniMax-M3-NVFP4 -# checkpoint. MiniMax-M3 modelopt NVFP4 support (vllm-project/vllm PR #46380) is -# baked into the perf container image, so no runtime patch is needed. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# launch_b200-nscale-slurm.sh rewrites MODEL to the pre-downloaded path; only download -# when handed a bare HF id (b200-cw / b200-nb runners). -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -start_gpu_monitor - -CONC_ARGS="" -if [ "$CONC" -lt 64 ]; then - CONC_ARGS="--no-enable-chunked-prefill --attention_config.minimax_m3_msa_decode_backend cutlass" -fi - -set -x -vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---attention_config.indexer_kv_dtype fp8 \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---kv-cache-dtype fp8 \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code \ -$CONC_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_b200_mtp.sh deleted file mode 100755 index 5082b6e923..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_b200_mtp.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 NVFP4 B200 single-node vLLM recipe with EAGLE3 speculative -# decoding — same shape as minimaxm3_fp8_b200_mtp.sh but uses the -# nvidia/MiniMax-M3-NVFP4 checkpoint. MiniMax-M3 modelopt NVFP4 support -# (vllm-project/vllm PR #46380) is baked into the perf container image, so no -# runtime patch is needed. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -# launch_b200-nscale-slurm.sh rewrites MODEL to the pre-downloaded path; only download -# the target when handed a bare HF id (b200-cw / b200-nb runners). The EAGLE3 -# draft is never pre-staged, so fetch it either way: next to the target weights -# when MODEL is a local path, into the HF cache otherwise. -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" - hf download "$DRAFT_MODEL" - DRAFT_MODEL_PATH="$DRAFT_MODEL" -else - DRAFT_MODEL_PATH="$(dirname "$MODEL")/${DRAFT_MODEL##*/}" - if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" - fi -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# Speculative-token count is picked per operating point to trace the -# Total-TPS/GPU vs median-interactivity Pareto frontier of the EAGLE3 offline -# sweep (bench_results_b200.json). The best num_speculative_tokens is not -# constant: fewer tokens win at the throughput end (high concurrency), more -# tokens win at the latency end (low concurrency). 3 is the default and the -# table below overrides the non-3 operating points. -# Key: ISL:TP:EP_SIZE:DP_ATTENTION:CONC (exactly the env vars the launcher sets; -# for dp-attn configs TP holds the data-parallel size, as in PARALLEL_ARGS below). -declare -A NUM_SPEC_TOKENS_MAP=( - # --- ISL=1024 / OSL=1024 --- - [1024:4:4:false:4]=4 - [1024:8:1:false:2]=6 - [1024:8:1:false:4]=4 - # --- ISL=8192 / OSL=1024 --- - [8192:2:1:false:1]=4 - [8192:2:1:false:256]=4 - [8192:2:2:false:1]=4 - [8192:2:2:false:16]=4 - [8192:2:2:false:32]=2 - [8192:2:2:false:64]=4 - [8192:2:2:false:128]=4 - [8192:2:2:false:256]=4 - [8192:2:2:false:512]=2 - [8192:4:1:false:256]=2 - [8192:4:4:false:256]=4 - [8192:8:1:false:1]=4 - [8192:8:1:false:2]=4 -) -NUM_SPEC_TOKENS="${NUM_SPEC_TOKENS_MAP[${ISL}:${TP}:${EP_SIZE}:${DP_ATTENTION}:${CONC}]:-3}" -echo "Selected NUM_SPEC_TOKENS=$NUM_SPEC_TOKENS for ISL=$ISL TP=$TP EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION CONC=$CONC" - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.9 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_b300.sh deleted file mode 100755 index 435fc25369..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_b300.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 NVFP4 B300 single-node vLLM recipe. -# Same shape as minimaxm3_fp8_b300.sh but uses the nvidia/MiniMax-M3-NVFP4 -# checkpoint. MiniMax-M3 modelopt NVFP4 support (vllm-project/vllm PR #46380) is -# baked into the perf container image, so no runtime patch is needed. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -start_gpu_monitor - -CONC_ARGS="" -if [ "$CONC" -lt 64 ]; then - CONC_ARGS="--no-enable-chunked-prefill --attention_config.minimax_m3_msa_decode_backend cutlass" -fi - -set -x -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \ -$PARALLEL_ARGS \ ---attention_config.indexer_kv_dtype fp8 \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---kv-cache-dtype fp8 \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code \ -$CONC_ARGS \ -> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_b300_mtp.sh deleted file mode 100755 index 3a89f14b82..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_b300_mtp.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 NVFP4 B300 single-node vLLM recipe with EAGLE3 speculative -# decoding — same shape as minimaxm3_fp8_b300_mtp.sh but uses the -# nvidia/MiniMax-M3-NVFP4 checkpoint. MiniMax-M3 modelopt NVFP4 support -# (vllm-project/vllm PR #46380) is baked into the perf container image, so no -# runtime patch is needed. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -# The target weights are launched from MODEL_PATH (the b300 launcher points it -# at the pre-staged read-only /scratch/models/MiniMax-M3-NVFP4). The EAGLE3 -# draft is not pre-staged and must be downloaded, so it cannot live next to the -# read-only target — fetch it into the writable models dir (/data/models) -# instead. When MODEL_PATH is unset (stand-alone runs) fall back to the HF cache. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi - DRAFT_MODEL_PATH="/data/models/${DRAFT_MODEL##*/}" - if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" - hf download "$DRAFT_MODEL" - DRAFT_MODEL_PATH="$DRAFT_MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# Speculative-token count is picked per operating point to trace the -# Total-TPS/GPU vs median-interactivity Pareto frontier of the EAGLE3 offline -# sweep (bench_results.json). The best num_speculative_tokens is not constant: -# fewer tokens win at the throughput end (high concurrency), more tokens win at -# the latency end (low concurrency). The frontier only ever selects 1..4 tokens, -# so 3 is the default and the table below overrides the non-3 operating points. -# Key: ISL:TP:EP_SIZE:DP_ATTENTION:CONC (exactly the env vars the launcher sets; -# for dp-attn configs TP holds the data-parallel size, as in PARALLEL_ARGS below). -declare -A NUM_SPEC_TOKENS_MAP=( - # --- ISL=1024 / OSL=1024 --- - [1024:2:1:false:512]=1 - [1024:2:2:false:512]=1 - [1024:2:2:false:1]=4 - [1024:4:4:false:2]=4 - [1024:8:1:false:1]=4 - # --- ISL=8192 / OSL=1024 --- - [8192:2:1:false:2]=2 - [8192:2:1:false:4]=4 - [8192:2:1:false:32]=2 - [8192:2:1:false:256]=2 - [8192:2:2:false:32]=4 - [8192:2:2:false:128]=2 - [8192:2:2:true:128]=4 - [8192:4:1:false:1]=4 - [8192:4:1:false:2]=4 - [8192:4:4:false:1]=4 - [8192:8:1:false:1]=4 -) -NUM_SPEC_TOKENS="${NUM_SPEC_TOKENS_MAP[${ISL}:${TP}:${EP_SIZE}:${DP_ATTENTION}:${CONC}]:-3}" -echo "Selected NUM_SPEC_TOKENS=$NUM_SPEC_TOKENS for ISL=$ISL TP=$TP EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION CONC=$CONC" - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -start_gpu_monitor - -set -x -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.9 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_mi355x_atom.sh deleted file mode 100644 index c35ec86d63..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_mi355x_atom.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -PARALLEL_ARGS=(-tp "$TP") #TP -if [ "$DP_ATTENTION" = "true" ]; then - if [ "$EP_SIZE" -gt 1 ]; then #DP+EP - PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) - else #DP+TP - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) - fi -fi - -SPEC_ARGS=() -OPT_ARGS=(--online_quant_config '{"global_quant_config": "ptpc_fp8", "exclude_layer": ["lm_head", "model.embed_tokens", "vision_tower", "multi_modal_projector", "patch_merge_mlp", "*block_sparse_moe"]}') - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.8 - -set -x -export AITER_QUICK_REDUCE_QUANTIZATION=INT4 -export ATOM_FORCE_ATTN_TRITON=1 -export MAX_MODEL_LEN=32768 -export MAX_NUM_BATCHED_TOKENS=32768 -export MAX_NUM_SEQS=256 -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${SPEC_ARGS[@]}" \ - "${OPT_ARGS[@]}" \ - --block-size 128 \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --max-model-len $MAX_MODEL_LEN \ - --max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS \ - --max-num-seqs $MAX_NUM_SEQS \ - --kv_cache_dtype fp8 \ - --trust-remote-code \ - --no-enable_prefix_caching \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code $( [[ ${#SPEC_ARGS[@]} -gt 0 ]] && echo "--use-chat-template" ) - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_mi355x_atom_mtp.sh deleted file mode 100644 index 1c60c8d2f9..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_mi355x_atom_mtp.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -PARALLEL_ARGS=(-tp "$TP") #TP -if [ "$DP_ATTENTION" = "true" ]; then - if [ "$EP_SIZE" -gt 1 ]; then #DP+EP - PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) - else #DP+TP - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) - fi -fi - -SPEC_ARGS=(--method eagle3 --draft-model Inferact/MiniMax-M3-EAGLE3 --num-speculative-tokens 3 ) -OPT_ARGS=(--online_quant_config '{"global_quant_config": "ptpc_fp8", "exclude_layer": ["lm_head", "model.embed_tokens", "vision_tower", "multi_modal_projector", "patch_merge_mlp", "*block_sparse_moe"]}') - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.8 - -set -x -export AITER_QUICK_REDUCE_QUANTIZATION=INT4 -export ATOM_FORCE_ATTN_TRITON=1 -export MAX_MODEL_LEN=32768 -export MAX_NUM_BATCHED_TOKENS=32768 -export MAX_NUM_SEQS=256 -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${SPEC_ARGS[@]}" \ - "${OPT_ARGS[@]}" \ - --block-size 128 \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --max-model-len $MAX_MODEL_LEN \ - --max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS \ - --max-num-seqs $MAX_NUM_SEQS \ - --kv_cache_dtype fp8 \ - --trust-remote-code \ - --no-enable_prefix_caching \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code $( [[ ${#SPEC_ARGS[@]} -gt 0 ]] && echo "--use-chat-template" ) - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_mi355x_vllm.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_mi355x_vllm.sh deleted file mode 100755 index f5ac074078..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_mi355x_vllm.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP4 MI355X (gfx950) single-node vLLM recipe. -# https://huggingface.co/amd/MiniMax-M3-MXFP4#reproduction -# Block size 128 is mandatory for MSA. This fixed-sequence benchmark uses the -# text-only language-model path with AITER MoE (vllm-project/vllm#46419). -# -# High-concurrency tuning (no model-architecture overrides): -# * INT4 quantized all-reduce (env knobs below) -- reduces the all-reduce -# cost (the biggest decode kernel); measured ~-12% to -17% TPOT at conc -# 64/128/256. -# * fp8 KV cache (--kv-cache-dtype fp8). - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 -export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 -# INT4 quantized all-reduce for the (~1.5 MB) decode all-reduces, which are the -# single biggest decode kernel at high concurrency. The MIN_SIZE_KB override is -# required: vLLM's default INT4 quick-reduce size gate for (bf16, TP4) is 16 MB, -# so it never fires for decode-sized tensors without it. -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --trust-remote-code \ - --block-size 128 \ - --no-enable-prefix-caching \ - --language-model-only \ - --max-model-len "$MAX_MODEL_LEN" \ - --attention-backend TRITON_ATTN \ - --moe-backend aiter \ - --kv-cache-dtype fp8 \ - --tool-call-parser minimax_m3 \ - --enable-auto-tool-choice \ - --reasoning-parser minimax_m3 > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_mi355x_vllm_mtp.sh deleted file mode 100755 index 69f1d71fa7..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp4_mi355x_vllm_mtp.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP4 MI355X (gfx950) single-node vLLM recipe with EAGLE3 -# speculative decoding. This is the spec-decoding=mtp variant of -# minimaxm3_fp4_mi355x_vllm.sh and uses three speculative tokens from -# Inferact/MiniMax-M3-EAGLE3. The pinned nightly includes upstream AMD -# MiniMax-M3 SupportsEagle3 support, so no runtime model patch is needed. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi -hf download "$DRAFT_MODEL" - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -# Use AITER MoE for the MXFP4 experts, matching minimaxm3_fp4_mi355x_vllm.sh. -# This is required for ALL configs including expert parallelism: with EP enabled -# and moe_backend=auto, the AITER MXFP4 backend is skipped and selection falls -# back to Mxfp4MoeBackend.EMULATION, which triggers a first-time build of the -# Quark hw-emulation C++ kernel (kernel_ext, 9 ROCm arches) on every worker at -# warmup. Concurrent EP workers deadlock on the shared torch_extensions build -# lock, hanging engine-core for hours. Forcing --moe-backend aiter selects the -# AITER_MXFP4_MXFP4 backend instead (verified working under TP4+EP4 with EAGLE3 -# spec decoding), avoiding the emulation build entirely. -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 -export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -NUM_SPEC_TOKENS=3 - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --trust-remote-code \ - --block-size 128 \ - --no-enable-prefix-caching \ - --language-model-only \ - --max-model-len "$MAX_MODEL_LEN" \ - --attention-backend TRITON_ATTN \ - --moe-backend aiter \ - --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ - --tool-call-parser minimax_m3 \ - --enable-auto-tool-choice \ - --reasoning-parser minimax_m3 > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_b200.sh deleted file mode 100755 index 89dc91a5b6..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_b200.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 B200 single-node vLLM recipe -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). 427B/26B-active MoE with MSA -# sparse attention. --block-size 128 is mandatory (MSA sparse_block_size is -# 128; the default 16 misaligns sparse indexing). The benchmark is text-only, -# so --language-model-only skips the vision encoder and frees VRAM for KV. -# dp-attn=true maps to DP×EP (DEP) per the recipe's "DP8 + Expert Parallel" -# layout; ep>1 maps to TP+EP (TEP). - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# launch_b200-nscale-slurm.sh rewrites MODEL to the pre-downloaded -# /lustre/fsw/gharunners/models/MiniMax-M3-MXFP8 path; only download when -# handed a bare HF id (b200-cw / b200-nb runners). -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -# 444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---attention-config '{"backend": "FLASHINFER", "use_trtllm_attention": true}' \ ---attention-config.indexer_kv_dtype "fp8" \ ---kv-cache-dtype fp8 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 32 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_b200_mtp.sh deleted file mode 100644 index 0edcdda836..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_b200_mtp.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 B200 single-node vLLM recipe with EAGLE3 speculative -# decoding — the repo's spec-decoding=mtp variant of minimaxm3_fp8_b200.sh -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). Adds the -# Inferact/MiniMax-M3-EAGLE3-GQA draft head via --speculative-config with 3 -# speculative tokens. Everything else keeps the non-MTP serve shape: -# --block-size 128 is mandatory (MSA sparse_block_size is 128; the default 16 -# misaligns sparse indexing), and --language-model-only skips the vision -# encoder for the text-only benchmark. dp-attn=true maps to DP×EP (DEP); -# ep>1 maps to TP+EP (TEP). -# -# The target uses the FlashInfer TRT-LLM attention path. The EAGLE3-GQA drafter -# is pinned separately to FLASH_ATTN. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# launch_b200-nscale-slurm.sh rewrites MODEL to the pre-downloaded -# /lustre/fsw/gharunners/models/MiniMax-M3-MXFP8 path; only download the target -# when handed a bare HF id (b200-cw / b200-nb runners). The EAGLE3 draft is -# never pre-staged, so fetch it either way: next to the target weights when -# MODEL is a local path (the gharunners tree is writable), into the HF cache -# otherwise. -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" - hf download "$DRAFT_MODEL" - DRAFT_MODEL_PATH="$DRAFT_MODEL" -else - DRAFT_MODEL_PATH="$(dirname "$MODEL")/${DRAFT_MODEL##*/}" - if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" - fi -fi - -SERVER_LOG=/workspace/server.log - -# 444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---attention-config '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' \ ---kv-cache-dtype fp8 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" \ ---stream-interval 32 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# Spec-decode acceptance rate degrades on raw random tokens; route prompts -# through the chat template as the other MTP recipes do. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_b300.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_b300.sh deleted file mode 100755 index e146fe4fb3..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_b300.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 B300 single-node vLLM recipe -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). Same shape as the B200 -# script, but follows the b300 launcher's MODEL/MODEL_PATH split: -# launch_b300-nv.sh keeps MODEL as the HF id and points MODEL_PATH at -# /data/models/ (writable NFS) for models not in the SRE-staged -# /scratch/models list — MiniMax-M3 is not staged. --block-size 128 is -# mandatory (MSA sparse/index cache); the benchmark is text-only, so -# --language-model-only frees the vision encoder's VRAM. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE. -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -# 444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---attention-config '{"backend": "FLASHINFER", "use_trtllm_attention": true}' \ ---attention-config.indexer_kv_dtype "fp8" \ ---kv-cache-dtype fp8 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 32 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_b300_mtp.sh deleted file mode 100644 index 7439387f9f..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_b300_mtp.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 B300 single-node vLLM recipe with EAGLE3 speculative -# decoding — the repo's spec-decoding=mtp variant of minimaxm3_fp8_b300.sh -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). Adds the -# Inferact/MiniMax-M3-EAGLE3-GQA draft head via --speculative-config with 3 -# speculative tokens. Everything else keeps the non-MTP serve shape: -# --block-size 128 is mandatory (MSA sparse/index cache); the benchmark is -# text-only, so --language-model-only frees the vision encoder's VRAM. -# -# The target uses the FlashInfer TRT-LLM attention path. The EAGLE3-GQA drafter -# is pinned separately to FLASH_ATTN. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE. -# Either way, MODEL_PATH is what the server is launched with. The EAGLE3 -# draft follows the same split: it lands next to the target weights (writable -# /data/models on b300 via the launcher's MODEL/MODEL_PATH split) when -# MODEL_PATH is set, in the HF cache otherwise. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi - DRAFT_MODEL_PATH="$(dirname "$MODEL_PATH")/${DRAFT_MODEL##*/}" - if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" - hf download "$DRAFT_MODEL" - DRAFT_MODEL_PATH="$DRAFT_MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -# 444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---attention-config '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' \ ---kv-cache-dtype fp8 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" \ ---stream-interval 32 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# Spec-decode acceptance rate degrades on raw random tokens; route prompts -# through the chat template as the other MTP recipes do. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_h100.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_h100.sh deleted file mode 100755 index 9d9c5802b1..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_h100.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 H100 single-node vLLM recipe -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). 427B/26B-active MoE with MSA -# sparse attention. --block-size 128 is mandatory (MSA sparse_block_size is -# 128; the default 16 misaligns sparse indexing). The benchmark is text-only, -# so --language-model-only skips the vision encoder and frees VRAM for KV. -# dp-attn=true maps to DP×EP (DEP) per the recipe's "DP8 + Expert Parallel" -# layout; ep>1 maps to TP+EP (TEP). Hopper has no native MX tensor cores, so -# the MXFP8 MoE runs through vLLM's Hopper-compatible backends (Marlin / -# DeepGEMM) selected by the mxfp8 oracle in the minimax-m3 image. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# The shared HF cache lives on a network FS; concurrent day-zero downloads of -# the same ~444 GB checkpoint from sibling nodes hit huggingface_hub's -# WeakFileLock "[Errno 116] Stale file handle" race. Retry the download (it -# resumes), then serve with HF_HUB_OFFLINE=1 so vllm's snapshot_download does -# a lock-free local-cache read instead of re-contending the lock files. -SERVE_OFFLINE=() -if [[ "$MODEL" != /* ]]; then - for attempt in 1 2 3 4 5; do - hf download "$MODEL" && break - if [ "$attempt" = 5 ]; then echo "hf download failed after $attempt attempts" >&2; exit 1; fi - echo "hf download attempt $attempt failed; retrying in 60s" >&2 - sleep 60 - done - SERVE_OFFLINE=(env HF_HUB_OFFLINE=1) -fi - -SERVER_LOG=/workspace/server.log - -export PYTHONNOUSERSITE=1 -# ~444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# Fixed-seq-len runs don't need graphs past the request concurrency: capture -# up to the next power of two >= CONC (per-DP-rank batch is CONC/DP but ragged -# arrival makes the full CONC bound safer), capped at vLLM's 2048 ceiling. -CAPTURE_SIZE=4 -while (( CAPTURE_SIZE < CONC )); do CAPTURE_SIZE=$((CAPTURE_SIZE * 2)); done -(( CAPTURE_SIZE > 2048 )) && CAPTURE_SIZE=2048 - -# H100 DEP is weights-bound: every DP rank replicates the ~20 GB -# BF16-dequantized attention/dense/embedding weights next to its ~52 GB -# expert shard, and at gmu 0.90 KV-cache init fails outright at high conc -# (sweep 27441767143, conc 256: "No available memory for the cache blocks"). -# Claw back headroom: higher gpu-memory-utilization, and decode graphs -# capped at 2x the per-rank batch share instead of the full CONC bound. -GMU=0.90 -if [ "${DP_ATTENTION}" = "true" ]; then - GMU=0.94 - PER_RANK_BOUND=$(( 2 * ((CONC + TP - 1) / TP) )) - CAPTURE_SIZE=4 - while (( CAPTURE_SIZE < PER_RANK_BOUND )); do CAPTURE_SIZE=$((CAPTURE_SIZE * 2)); done - (( CAPTURE_SIZE > 2048 )) && CAPTURE_SIZE=2048 -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -"${SERVE_OFFLINE[@]}" vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization $GMU \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size $CAPTURE_SIZE \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_h100_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_h100_mtp.sh deleted file mode 100644 index caa57e432b..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_h100_mtp.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 H100 single-node vLLM recipe with EAGLE3 speculative -# decoding — the repo's spec-decoding=mtp variant of minimaxm3_fp8_h100.sh -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). Adds the -# Inferact/MiniMax-M3-EAGLE3 draft head via --speculative-config with 3 -# speculative tokens. Everything else keeps the non-MTP serve shape: -# --block-size 128 is mandatory (MSA sparse_block_size is 128), the benchmark -# is text-only so --language-model-only frees the vision encoder's VRAM, and -# the MXFP8 MoE runs through vLLM's Hopper-compatible backends. H100 is -# TP8-only (~56 GB of weights per 80 GB GPU below TP8 leaves no room). -# -# The drafter is pinned to FLASH_ATTN: the EAGLE3 head is MHA, and FlashInfer -# only supports page size 128 through its trtllm-gen kernel, which requires -# GQA/MQA — engine init dies in FlashInferMetadataBuilder otherwise (hit on -# the B300 MTP canary). FLASH_ATTN takes any multiple-of-16 block size, so -# the mandatory 128 is fine for the draft. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# The shared HF cache lives on a network FS; concurrent day-zero downloads of -# the same ~444 GB checkpoint from sibling nodes hit huggingface_hub's -# WeakFileLock "[Errno 116] Stale file handle" race. Retry the download (it -# resumes), then serve with HF_HUB_OFFLINE=1 so vllm's snapshot_download does -# a lock-free local-cache read instead of re-contending the lock files. The -# EAGLE3 draft is fetched the same way so the offline serve finds it cached. -SERVE_OFFLINE=() -if [[ "$MODEL" != /* ]]; then - for attempt in 1 2 3 4 5; do - hf download "$MODEL" && break - if [ "$attempt" = 5 ]; then echo "hf download failed after $attempt attempts" >&2; exit 1; fi - echo "hf download attempt $attempt failed; retrying in 60s" >&2 - sleep 60 - done - for attempt in 1 2 3 4 5; do - hf download "$DRAFT_MODEL" && break - if [ "$attempt" = 5 ]; then echo "hf download failed after $attempt attempts" >&2; exit 1; fi - echo "hf download attempt $attempt failed; retrying in 60s" >&2 - sleep 60 - done - SERVE_OFFLINE=(env HF_HUB_OFFLINE=1) -fi - -SERVER_LOG=/workspace/server.log - -export PYTHONNOUSERSITE=1 -# ~444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -# Fixed-seq-len runs don't need graphs past the decode step's token count: -# with spec decoding every running request contributes 1 + NUM_SPEC_TOKENS -# tokens per step, so capture up to the next power of two >= -# CONC * (1 + NUM_SPEC_TOKENS), capped at vLLM's 2048 ceiling. -CAPTURE_SIZE=4 -while (( CAPTURE_SIZE < CONC * (1 + NUM_SPEC_TOKENS) )); do CAPTURE_SIZE=$((CAPTURE_SIZE * 2)); done -(( CAPTURE_SIZE > 2048 )) && CAPTURE_SIZE=2048 - -# H100 DEP is weights-bound: every DP rank replicates the ~20 GB -# BF16-dequantized attention/dense/embedding weights next to its ~52 GB -# expert shard, and at gmu 0.90 KV-cache init fails outright at high conc -# (sweep 27441767143, conc 256: "No available memory for the cache blocks"). -# Claw back headroom: higher gpu-memory-utilization, and decode graphs -# capped at 2x the per-rank batch share (spec-token-scaled) instead of the -# full CONC bound. -GMU=0.90 -if [ "${DP_ATTENTION}" = "true" ]; then - GMU=0.94 - PER_RANK_BOUND=$(( 2 * ((CONC + TP - 1) / TP) * (1 + NUM_SPEC_TOKENS) )) - CAPTURE_SIZE=4 - while (( CAPTURE_SIZE < PER_RANK_BOUND )); do CAPTURE_SIZE=$((CAPTURE_SIZE * 2)); done - (( CAPTURE_SIZE > 2048 )) && CAPTURE_SIZE=2048 -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -"${SERVE_OFFLINE[@]}" vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization $GMU \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size $CAPTURE_SIZE \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# Spec-decode acceptance rate degrades on raw random tokens; route prompts -# through the chat template as the other MTP recipes do. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_h200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_h200.sh deleted file mode 100755 index bf6ddb2c21..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_h200.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 H200 single-node vLLM recipe -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). 427B/26B-active MoE with MSA -# sparse attention. --block-size 128 is mandatory (MSA sparse_block_size is -# 128; the default 16 misaligns sparse indexing). The benchmark is text-only, -# so --language-model-only skips the vision encoder and frees VRAM for KV. -# dp-attn=true maps to DP×EP (DEP) per the recipe's "DP8 + Expert Parallel" -# layout; ep>1 maps to TP+EP (TEP). Hopper has no native MX tensor cores, so -# the MXFP8 MoE runs through vLLM's Hopper-compatible backends (Marlin / -# DeepGEMM) selected by the mxfp8 oracle in the minimax-m3 image. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# The shared HF cache lives on a network FS; concurrent day-zero downloads of -# the same ~444 GB checkpoint from sibling nodes hit huggingface_hub's -# WeakFileLock "[Errno 116] Stale file handle" race. Retry the download (it -# resumes), then serve with HF_HUB_OFFLINE=1 so vllm's snapshot_download does -# a lock-free local-cache read instead of re-contending the lock files. -SERVE_OFFLINE=() -if [[ "$MODEL" != /* ]]; then - for attempt in 1 2 3 4 5; do - hf download "$MODEL" && break - if [ "$attempt" = 5 ]; then echo "hf download failed after $attempt attempts" >&2; exit 1; fi - echo "hf download attempt $attempt failed; retrying in 60s" >&2 - sleep 60 - done - SERVE_OFFLINE=(env HF_HUB_OFFLINE=1) -fi - -SERVER_LOG=/workspace/server.log - -export PYTHONNOUSERSITE=1 -# ~444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# Fixed-seq-len runs don't need graphs past the request concurrency: capture -# up to the next power of two >= CONC (per-DP-rank batch is CONC/DP but ragged -# arrival makes the full CONC bound safer), capped at vLLM's 2048 ceiling. -CAPTURE_SIZE=4 -while (( CAPTURE_SIZE < CONC )); do CAPTURE_SIZE=$((CAPTURE_SIZE * 2)); done -(( CAPTURE_SIZE > 2048 )) && CAPTURE_SIZE=2048 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -"${SERVE_OFFLINE[@]}" vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size $CAPTURE_SIZE \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_h200_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_h200_mtp.sh deleted file mode 100644 index d94d5cc014..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_h200_mtp.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 H200 single-node vLLM recipe with EAGLE3 speculative -# decoding — the repo's spec-decoding=mtp variant of minimaxm3_fp8_h200.sh -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). Adds the -# Inferact/MiniMax-M3-EAGLE3 draft head via --speculative-config with 3 -# speculative tokens. Everything else keeps the non-MTP serve shape: -# --block-size 128 is mandatory (MSA sparse_block_size is 128), the benchmark -# is text-only so --language-model-only frees the vision encoder's VRAM, and -# the MXFP8 MoE runs through vLLM's Hopper-compatible backends. -# -# The drafter is pinned to FLASH_ATTN: the EAGLE3 head is MHA, and FlashInfer -# only supports page size 128 through its trtllm-gen kernel, which requires -# GQA/MQA — engine init dies in FlashInferMetadataBuilder otherwise (hit on -# the B300 MTP canary). FLASH_ATTN takes any multiple-of-16 block size, so -# the mandatory 128 is fine for the draft. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# The shared HF cache lives on a network FS; concurrent day-zero downloads of -# the same ~444 GB checkpoint from sibling nodes hit huggingface_hub's -# WeakFileLock "[Errno 116] Stale file handle" race. Retry the download (it -# resumes), then serve with HF_HUB_OFFLINE=1 so vllm's snapshot_download does -# a lock-free local-cache read instead of re-contending the lock files. The -# EAGLE3 draft is fetched the same way so the offline serve finds it cached. -SERVE_OFFLINE=() -if [[ "$MODEL" != /* ]]; then - for attempt in 1 2 3 4 5; do - hf download "$MODEL" && break - if [ "$attempt" = 5 ]; then echo "hf download failed after $attempt attempts" >&2; exit 1; fi - echo "hf download attempt $attempt failed; retrying in 60s" >&2 - sleep 60 - done - for attempt in 1 2 3 4 5; do - hf download "$DRAFT_MODEL" && break - if [ "$attempt" = 5 ]; then echo "hf download failed after $attempt attempts" >&2; exit 1; fi - echo "hf download attempt $attempt failed; retrying in 60s" >&2 - sleep 60 - done - SERVE_OFFLINE=(env HF_HUB_OFFLINE=1) -fi - -SERVER_LOG=/workspace/server.log - -export PYTHONNOUSERSITE=1 -# ~444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -# Fixed-seq-len runs don't need graphs past the decode step's token count: -# with spec decoding every running request contributes 1 + NUM_SPEC_TOKENS -# tokens per step, so capture up to the next power of two >= -# CONC * (1 + NUM_SPEC_TOKENS), capped at vLLM's 2048 ceiling. -CAPTURE_SIZE=4 -while (( CAPTURE_SIZE < CONC * (1 + NUM_SPEC_TOKENS) )); do CAPTURE_SIZE=$((CAPTURE_SIZE * 2)); done -(( CAPTURE_SIZE > 2048 )) && CAPTURE_SIZE=2048 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -"${SERVE_OFFLINE[@]}" vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size $CAPTURE_SIZE \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# Spec-decode acceptance rate degrades on raw random tokens; route prompts -# through the chat template as the other MTP recipes do. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi300x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi300x.sh deleted file mode 100755 index 917cf78352..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi300x.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 MI300X (gfx942) single-node vLLM recipe. -# Reuses the dedicated ROCm image and the MI355X serving shape. Block size 128 -# is mandatory for MSA sparse attention. Keep the default BF16 KV cache on -# gfx942: the checkpoint has no calibrated q/prob scales for ROCm FP8 -# attention, and vLLM's fallback scale of 1.0 corrupts model accuracy. -# --async-scheduling and --max-num-batched-tokens 16384 are accuracy-safe -# scheduling levers: the larger prefill batch amortizes the per-step -# BF16-emulated MoE weight read, lifting high-concurrency throughput. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 - -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MHA=0 - -export TORCH_BLAS_PREFER_HIPBLASLT=1 -export NCCL_MIN_NCHANNELS="${NCCL_MIN_NCHANNELS:-112}" -export GPU_MAX_HW_QUEUES="${GPU_MAX_HW_QUEUES:-2}" - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --block-size 128 \ - --no-enable-prefix-caching \ - --language-model-only \ - --max-model-len "$MAX_MODEL_LEN" \ - --attention-backend TRITON_ATTN \ - --async-scheduling \ - --max-num-batched-tokens 16384 \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi300x_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi300x_mtp.sh deleted file mode 100644 index 6de6422ed5..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi300x_mtp.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 MI300X (gfx942) single-node vLLM recipe with EAGLE3 -# speculative decoding — the spec-decoding=mtp variant of -# minimaxm3_fp8_mi300x.sh. Adds the Inferact/MiniMax-M3-EAGLE3 draft head via -# --speculative-config with 3 speculative tokens. Everything else mirrors the -# non-MTP MI300X recipe: mandatory --block-size 128, --language-model-only for -# the text-only benchmark, --attention-backend TRITON_ATTN, and -# --no-enable-prefix-caching. Runs with CUDA graphs (no --enforce-eager); -# VLLM_USE_BREAKABLE_CUDAGRAPH=0 avoids the M3-decode breakable-cudagraph path. -# FP8 KV cache reduces memory pressure and increases concurrency headroom. -# -# Unlike the CUDA recipes, the drafter needs no attention_backend override: -# the FlashInfer "page size 128 requires GQA/MQA" limitation that forced -# FLASH_ATTN for the EAGLE3 MHA head on Blackwell is FlashInfer/CUDA-specific. -# Here the whole server runs on TRITON_ATTN (set globally below), which serves -# the MHA draft fine. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -# MODEL is a bare HF id on the mi300x single-node runner (a fast cache hit when -# pre-staged). The EAGLE3 draft is not staged; fetch it into the same cache. -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" - hf download "$DRAFT_MODEL" -fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --block-size 128 \ - --kv-cache-dtype fp8 \ - --no-enable-prefix-caching \ - --language-model-only \ - --max-model-len "$MAX_MODEL_LEN" \ - --attention-backend TRITON_ATTN \ - --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# Spec-decode acceptance rate degrades on raw random tokens; route prompts -# through the chat template as the other MTP recipes do. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi325x.sh deleted file mode 100755 index 48e83cfa73..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi325x.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 MI325X (gfx942) single-node vLLM recipe. -# https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3?hardware=mi325x&variant=mxfp8 -# MXFP8 runs from TP=4 on gfx942; block size 128 is mandatory for MSA. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --block-size 128 \ - --language-model-only \ - --max-model-len "$MAX_MODEL_LEN" \ - --attention-backend TRITON_ATTN \ - --no-enable-prefix-caching \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi325x_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi325x_mtp.sh deleted file mode 100644 index 24d4eae974..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi325x_mtp.sh +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 MI325X (gfx942) single-node vLLM recipe with EAGLE3 -# speculative decoding — the spec-decoding=mtp variant of -# minimaxm3_fp8_mi325x.sh. Adds the Inferact/MiniMax-M3-EAGLE3 draft head via -# --speculative-config with 3 speculative tokens. Everything else mirrors the -# non-MTP MI325X recipe: mandatory --block-size 128, --language-model-only for -# the text-only benchmark, --attention-backend TRITON_ATTN, and -# --no-enable-prefix-caching. Runs with CUDA graphs (no --enforce-eager); -# VLLM_USE_BREAKABLE_CUDAGRAPH=0 avoids the M3-decode breakable-cudagraph path. -# The default BF16 KV cache is retained (unlike the MI355X recipe's FP8 KV -# cache): gfx942 has no calibrated q/prob scales for ROCm FP8 attention and -# vLLM's fallback scale of 1.0 corrupts accuracy. -# -# Unlike the CUDA recipes, the drafter needs no attention_backend override: -# the FlashInfer "page size 128 requires GQA/MQA" limitation that forced -# FLASH_ATTN for the EAGLE3 MHA head on Blackwell is FlashInfer/CUDA-specific. -# Here the whole server runs on TRITON_ATTN (set globally below), which serves -# the MHA draft fine. -# -# [AI generated draft test] The shipped vllm/vllm-openai-rocm:minimax-m3 image -# does NOT implement SupportsEagle3 on the AMD MiniMax-M3 model, so EAGLE3 -# engine init fails with "Model does not support EAGLE3 interface but -# aux_hidden_state_outputs was requested". This recipe applies that fix -# (functionstackx/vllm#1 — ported from nvidia/model.py, upstreamed as -# vllm-project/vllm#45546) in-place to the installed vllm before serving, so we -# can validate EAGLE3 on real MI325X hardware ahead of an image rebuild. The -# same patch is validated green on MI355X. It is idempotent and fails the job -# loudly if the installed amd/model.py has drifted from the expected base. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -# MODEL is a bare HF id on the mi325x single-node runner (a fast cache hit when -# pre-staged). The EAGLE3 draft is not staged; fetch it into the same cache. -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" - hf download "$DRAFT_MODEL" -fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -# [AI generated draft test] Patch the installed AMD MiniMax-M3 model to add the -# SupportsEagle3 interface (functionstackx/vllm#1, upstream vllm-project/vllm#45546). -# Mirrors nvidia/model.py: adds EagleModelMixin to the inner model + -# aux-hidden-state emission, and SupportsEagle3 to the two outer classes. -# Idempotent; hard-fails if the installed file has drifted from the expected -# base (so we never silently run unpatched and mislabel the result). -python3 - <<'PYEOF' || { echo "EAGLE3 in-place patch failed" >&2; exit 1; } -import ast, importlib.util, pathlib, sys - -spec = importlib.util.find_spec("vllm") -root = pathlib.Path(spec.submodule_search_locations[0]) -target = root / "models" / "minimax_m3" / "amd" / "model.py" -src = target.read_text() - -if "EagleModelMixin" in src and "class MiniMaxM3Model(nn.Module, EagleModelMixin):" in src: - print(f"[eagle3-patch] already applied: {target}") - sys.exit(0) - -edits = [ - ( - "from vllm.model_executor.models.interfaces import (\n" - " MultiModalEmbeddings,\n" - " SupportsMultiModal,\n" - ")", - "from vllm.model_executor.models.interfaces import (\n" - " EagleModelMixin,\n" - " MultiModalEmbeddings,\n" - " SupportsEagle3,\n" - " SupportsMultiModal,\n" - ")", - ), - ( - "class MiniMaxM3Model(nn.Module):", - "class MiniMaxM3Model(nn.Module, EagleModelMixin):", - ), - ( - " inputs_embeds: torch.Tensor | None = None,\n" - " ) -> torch.Tensor:\n" - " if inputs_embeds is not None:", - " inputs_embeds: torch.Tensor | None = None,\n" - " ) -> torch.Tensor | tuple[torch.Tensor, list[torch.Tensor]]:\n" - " if inputs_embeds is not None:", - ), - ( - " residual = None\n\n" - " for layer in self.layers[self.start_layer : self.end_layer]:\n" - " hidden_states, residual = layer(positions, hidden_states, residual)\n\n" - " hidden_states, _ = self.norm(hidden_states, residual)\n" - " return hidden_states", - " residual = None\n\n" - " # EAGLE3 is not yet compatible with pipeline parallel\n" - " aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)\n" - " for idx, layer in enumerate(self.layers[self.start_layer : self.end_layer]):\n" - " hidden_states, residual = layer(positions, hidden_states, residual)\n" - " self._maybe_add_hidden_state(\n" - " aux_hidden_states, idx + 1, hidden_states, residual\n" - " )\n\n" - " hidden_states, _ = self.norm(hidden_states, residual)\n\n" - " if len(aux_hidden_states) > 0:\n" - " return hidden_states, aux_hidden_states\n" - " return hidden_states", - ), - ( - "class MiniMaxM3SparseForCausalLM(nn.Module):", - "class MiniMaxM3SparseForCausalLM(nn.Module, SupportsEagle3):", - ), - ( - "class MiniMaxM3SparseForConditionalGeneration(nn.Module, SupportsMultiModal):", - "class MiniMaxM3SparseForConditionalGeneration(\n" - " nn.Module, SupportsMultiModal, SupportsEagle3\n" - "):", - ), -] - -for old, new in edits: - count = src.count(old) - if count != 1: - sys.exit( - f"[eagle3-patch] anchor matched {count} times (expected 1); " - f"installed {target} has drifted from the expected base — aborting" - ) - src = src.replace(old, new) - -ast.parse(src) -target.write_text(src) -print(f"[eagle3-patch] applied EAGLE3 support to {target}") -PYEOF - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --block-size 128 \ - --no-enable-prefix-caching \ - --language-model-only \ - --max-model-len "$MAX_MODEL_LEN" \ - --attention-backend TRITON_ATTN \ - --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# Spec-decode acceptance rate degrades on raw random tokens; route prompts -# through the chat template as the other MTP recipes do. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi355x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi355x.sh deleted file mode 100755 index 7df628b70b..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi355x.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 MI355X (gfx950) single-node vLLM recipe. -# https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 -# The recipe recommends MXFP8 from TP=4 on gfx950 and requires block size 128. -# -# AITER page-16 sparse paged-attention fast path (vllm-project/vllm#47287, -# merged into the pinned nightly): maps MiniMax-M3's top-k 128-token sparse -# blocks onto AITER page-16 block tables and runs AITER Gluon paged attention -# over only the selected KV pages. This is a kernel-level speedup of the same -# sparse-attention computation (no FLOP reduction), enabled via -# VLLM_ROCM_USE_AITER=1 + VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 with fp8 KV cache -# on a TP where each rank has num_kv_heads == 1 (TP4). We deliberately do NOT -# pass the #47269 --hf-overrides use_index_cache/index_topk_freq cross-layer -# indexer-skip override: it reduces model-architecture FLOPs, which is -# disallowed by docs/PR_REVIEW_CHECKLIST.md. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -# MI355X mxfp8 recipe (vllm-project/recipes#581): INT4 quick all-reduce plus -# the router-append shared-experts MoE fusion (vllm-project/vllm#46545). INT4 -# quick all-reduce is applied at all concurrencies (accuracy is guarded by the -# 8k1k evals); #2003 used INT6. -export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 - -# AITER page-16 sparse PA (vllm-project/vllm#47287) is a long-context, -# high-concurrency optimization: it maps MiniMax-M3's top-k 128-token sparse -# blocks onto AITER page-16 block tables. Measured on gfx950 MXFP8, it only wins -# in the 8k1k high-concurrency tail and adds overhead at short context (1k1k) or -# low batch. So enable the "high-conc fast path" (shuffled KV-cache layout for -# sparse PA + the emulation dense-linear backend, see below) only for -# isl>=8192 && conc>=64; everywhere else fall back to the #2003 path -# (non-shuffled Triton attention + native linear). Overridable via -# MM3_HIGH_CONC_FASTPATH=0/1. -if [ -z "${MM3_HIGH_CONC_FASTPATH:-}" ]; then - if [ "$ISL" -ge 8192 ] && [ "$CONC" -ge 64 ]; then - MM3_HIGH_CONC_FASTPATH=1 - else - MM3_HIGH_CONC_FASTPATH=0 - fi -fi - -if [ "$MM3_HIGH_CONC_FASTPATH" = "1" ]; then - export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 - # Quick all-reduce tuning from the MiniMax-M3 AITER recipe (vllm-project/vllm#47287): - # keep the bf16 accumulation and only quantize all-reduces above 256 KB. - export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0 - export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256 -else - export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=0 -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -# Previously when EP is On, VLLM_ROCM_USE_AITER needs to be off. -# After https://github.com/vllm-project/vllm/pull/47158, -# it can be simplified as VLLM_ROCM_USE_AITER=1. -# As the configs are TP only, remove the conditional check. -export VLLM_ROCM_USE_AITER=1 - -# Larger per-step prefill token budget to improve TP4 throughput at high -# concurrency. Overridable via env. -MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-32768}" - -# Dense-linear backend, gated on the same high-conc fast path as sparse PA. On -# this nightly the native Triton MXFP8 linear GEMM wins in the memory-bound -# low-concurrency regime, while --linear-backend emulation (bf16 hipBLASLT) wins -# in the compute-bound high-concurrency regime (~+3-5% at 8k1k conc>=64). -# LINEAR_BACKEND overrides (a backend name to force it, or "native" to disable). -LINEAR_ARGS=() -if [ -n "${LINEAR_BACKEND:-}" ]; then - [ "$LINEAR_BACKEND" != "native" ] && LINEAR_ARGS=(--linear-backend "$LINEAR_BACKEND") -elif [ "$MM3_HIGH_CONC_FASTPATH" = "1" ]; then - LINEAR_ARGS=(--linear-backend emulation) -fi - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --block-size 128 \ - --no-enable-prefix-caching \ - --language-model-only \ - --moe-backend aiter \ - --max-model-len "$MAX_MODEL_LEN" \ - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" \ - --kv-cache-dtype fp8 \ - --attention-backend TRITON_ATTN \ - "${LINEAR_ARGS[@]}" \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi355x_atom.sh deleted file mode 100644 index 6bbe70d5d0..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi355x_atom.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -PARALLEL_ARGS=(-tp "$TP") #TP -if [ "$DP_ATTENTION" = "true" ]; then - if [ "$EP_SIZE" -gt 1 ]; then #DP+EP - PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) - else #DP+TP - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) - fi -fi - -SPEC_ARGS=() -OPT_ARGS=(--online_quant_config '{"global_quant_config": "ptpc_fp8", "exclude_layer": ["lm_head", "model.embed_tokens", "vision_tower", "multi_modal_projector", "patch_merge_mlp", "*block_sparse_moe"]}') - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.8 - -set -x -export AITER_QUICK_REDUCE_QUANTIZATION=INT4 -export ATOM_FORCE_ATTN_TRITON=1 -export MAX_MODEL_LEN=32768 -export MAX_NUM_BATCHED_TOKENS=32768 -export MAX_NUM_SEQS=256 -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${SPEC_ARGS[@]}" \ - "${OPT_ARGS[@]}" \ - --block-size 128 \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --max-model-len $MAX_MODEL_LEN \ - --max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS \ - --max-num-seqs $MAX_NUM_SEQS \ - --kv_cache_dtype fp8 \ - --trust-remote-code \ - --no-enable_prefix_caching \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code $( [[ ${#SPEC_ARGS[@]} -gt 0 ]] && echo "--use-chat-template" ) - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi355x_atom_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi355x_atom_mtp.sh deleted file mode 100644 index a6bb3321d2..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi355x_atom_mtp.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -PARALLEL_ARGS=(-tp "$TP") #TP -if [ "$DP_ATTENTION" = "true" ]; then - if [ "$EP_SIZE" -gt 1 ]; then #DP+EP - PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) - else #DP+TP - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) - fi -fi - -SPEC_ARGS=(--method eagle3 --draft-model Inferact/MiniMax-M3-EAGLE3 --num-speculative-tokens 3 ) -OPT_ARGS=(--online_quant_config '{"global_quant_config": "ptpc_fp8", "exclude_layer": ["lm_head", "model.embed_tokens", "vision_tower", "multi_modal_projector", "patch_merge_mlp", "*block_sparse_moe"]}') - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.8 - -set -x -export AITER_QUICK_REDUCE_QUANTIZATION=INT4 -export ATOM_FORCE_ATTN_TRITON=1 -export MAX_MODEL_LEN=32768 -export MAX_NUM_BATCHED_TOKENS=32768 -export MAX_NUM_SEQS=256 -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${SPEC_ARGS[@]}" \ - "${OPT_ARGS[@]}" \ - --block-size 128 \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --max-model-len $MAX_MODEL_LEN \ - --max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS \ - --max-num-seqs $MAX_NUM_SEQS \ - --kv_cache_dtype fp8 \ - --trust-remote-code \ - --no-enable_prefix_caching \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code $( [[ ${#SPEC_ARGS[@]} -gt 0 ]] && echo "--use-chat-template" ) - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi355x_mtp.sh deleted file mode 100644 index 6871e50fc0..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi355x_mtp.sh +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 MI355X (gfx950) single-node vLLM recipe with EAGLE3 -# speculative decoding — the spec-decoding=mtp variant of -# minimaxm3_fp8_mi355x.sh. Adds the Inferact/MiniMax-M3-EAGLE3 draft head via -# --speculative-config with 3 speculative tokens. -# -# The EAGLE3 drafter (dense Llama MHA head) is pinned to TRITON_ATTN in the -# speculative-config, otherwise it would fall back to a slow default backend. -# Adding the explicit override left the draft's token acceptance unchanged but -# sped up the draft forward enough to turn into a win across the board. -# -# [AI generated draft test] The shipped vllm/vllm-openai-rocm:minimax-m3 image -# does NOT implement SupportsEagle3 on the AMD MiniMax-M3 model, so EAGLE3 -# engine init fails with "Model does not support EAGLE3 interface but -# aux_hidden_state_outputs was requested". This recipe applies that fix -# (functionstackx/vllm#1 — ported from nvidia/model.py) in-place to the -# installed vllm before serving, so we can validate EAGLE3 on real MI355X -# hardware ahead of an image rebuild. The patch is idempotent and fails the -# job loudly if the installed amd/model.py has drifted from the expected base. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="${DRAFT_MODEL:-Inferact/MiniMax-M3-EAGLE3}" - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -# MODEL stays a bare HF id on the mi355x single-node runner (weights are -# pre-staged in the mounted NFS HF cache, so this is a fast cache hit). The -# EAGLE3 draft is not staged; fetch it into the same cache. -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" - hf download "$DRAFT_MODEL" -fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -# Run with CUDA graphs (no --enforce-eager): VLLM_USE_BREAKABLE_CUDAGRAPH=0 -# avoids the M3-decode breakable-cudagraph path that previously forced eager. -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -# MI355X mxfp8 recipe (vllm-project/recipes#581): INT6 quick all-reduce plus -# the router-append shared-experts MoE fusion (vllm-project/vllm#46545). -export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT6 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -# Gate the AITER master switch on expert parallelism. With EP, -# the AITER master switch produces degenerate MiniMax-M3 -# output, so leave it off. -if printf '%s\n' "${PARALLEL_ARGS[@]}" | grep -qxF -- '--enable-expert-parallel'; then - export VLLM_ROCM_USE_AITER=0 -else - export VLLM_ROCM_USE_AITER=1 -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -# Larger per-step prefill token budget to improve TP4 throughput at high -# concurrency. Overridable via env. -MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-32768}" - -# [AI generated draft test] Patch the installed AMD MiniMax-M3 model to add the -# SupportsEagle3 interface (functionstackx/vllm#1). Mirrors nvidia/model.py: -# adds EagleModelMixin to the inner model + aux-hidden-state emission, and -# SupportsEagle3 to the two outer classes. Idempotent; hard-fails if the -# installed file has drifted from the expected base (so we never silently run -# unpatched and mislabel the result). -python3 - <<'PYEOF' || { echo "EAGLE3 in-place patch failed" >&2; exit 1; } -import ast, importlib.util, pathlib, sys - -spec = importlib.util.find_spec("vllm") -root = pathlib.Path(spec.submodule_search_locations[0]) -target = root / "models" / "minimax_m3" / "amd" / "model.py" -src = target.read_text() - -if "EagleModelMixin" in src and "class MiniMaxM3Model(nn.Module, EagleModelMixin):" in src: - print(f"[eagle3-patch] already applied: {target}") - sys.exit(0) - -edits = [ - ( - "from vllm.model_executor.models.interfaces import (\n" - " MultiModalEmbeddings,\n" - " SupportsMultiModal,\n" - ")", - "from vllm.model_executor.models.interfaces import (\n" - " EagleModelMixin,\n" - " MultiModalEmbeddings,\n" - " SupportsEagle3,\n" - " SupportsMultiModal,\n" - ")", - ), - ( - "class MiniMaxM3Model(nn.Module):", - "class MiniMaxM3Model(nn.Module, EagleModelMixin):", - ), - ( - " inputs_embeds: torch.Tensor | None = None,\n" - " ) -> torch.Tensor:\n" - " if inputs_embeds is not None:", - " inputs_embeds: torch.Tensor | None = None,\n" - " ) -> torch.Tensor | tuple[torch.Tensor, list[torch.Tensor]]:\n" - " if inputs_embeds is not None:", - ), - ( - " residual = None\n\n" - " for layer in self.layers[self.start_layer : self.end_layer]:\n" - " hidden_states, residual = layer(positions, hidden_states, residual)\n\n" - " hidden_states, _ = self.norm(hidden_states, residual)\n" - " return hidden_states", - " residual = None\n\n" - " # EAGLE3 is not yet compatible with pipeline parallel\n" - " aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)\n" - " for idx, layer in enumerate(self.layers[self.start_layer : self.end_layer]):\n" - " hidden_states, residual = layer(positions, hidden_states, residual)\n" - " self._maybe_add_hidden_state(\n" - " aux_hidden_states, idx + 1, hidden_states, residual\n" - " )\n\n" - " hidden_states, _ = self.norm(hidden_states, residual)\n\n" - " if len(aux_hidden_states) > 0:\n" - " return hidden_states, aux_hidden_states\n" - " return hidden_states", - ), - ( - "class MiniMaxM3SparseForCausalLM(nn.Module):", - "class MiniMaxM3SparseForCausalLM(nn.Module, SupportsEagle3):", - ), - ( - "class MiniMaxM3SparseForConditionalGeneration(nn.Module, SupportsMultiModal):", - "class MiniMaxM3SparseForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsEagle3):", - ), -] - -for old, new in edits: - count = src.count(old) - if count != 1: - sys.exit( - f"[eagle3-patch] anchor matched {count} times (expected 1); " - f"installed {target} has drifted from the expected base — aborting" - ) - src = src.replace(old, new) - -ast.parse(src) -target.write_text(src) -print(f"[eagle3-patch] applied EAGLE3 support to {target}") -PYEOF - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --block-size 128 \ - --no-enable-prefix-caching \ - --language-model-only \ - --moe-backend aiter \ - --max-model-len "$MAX_MODEL_LEN" \ - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" \ - --kv-cache-dtype fp8 \ - --attention-backend TRITON_ATTN \ - --linear-backend emulation \ - --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\"}" \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# Spec-decode acceptance rate degrades on raw random tokens; route prompts -# through the chat template as the other MTP recipes do. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_b200.sh deleted file mode 100755 index 422c2cdf38..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_b200.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -export PYTHONUNBUFFERED=1 - -SERVER_LOG=/workspace/server.log - -# Default: recv every ~10 requests; if CONC ≥ 16, relax to ~30 requests between scheduler recv polls. -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -MEM_FRAC_STATIC=0.82 -CHUNKED_PREFILL_SIZE=32768 -MAX_PREFILL_TOKENS=32768 -CUDA_GRAPH_MAX_BATCH_SIZE=$CONC -MAX_RUNNING_REQUESTS=128 -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---served-model-name "Qwen/Qwen3.5-397B-A17B" --trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 --ep-size $EP_SIZE \ ---cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ ---mem-fraction-static $MEM_FRAC_STATIC --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ ---context-length $CONTEXT_LENGTH --disable-radix-cache \ ---attention-backend trtllm_mha --moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---tokenizer-worker-num 6 --stream-interval 30 > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_b200_mtp.sh deleted file mode 100755 index 4c48d28fc5..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_b200_mtp.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -export PYTHONUNBUFFERED=1 - -SERVER_LOG=/workspace/server.log - -# Default: recv every ~10 requests; if CONC ≥ 16, relax to ~30 requests between scheduler recv polls. -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -MEM_FRAC_STATIC=0.82 -CHUNKED_PREFILL_SIZE=32768 -MAX_PREFILL_TOKENS=32768 -CUDA_GRAPH_MAX_BATCH_SIZE=$CONC -MAX_RUNNING_REQUESTS=128 -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---served-model-name "Qwen/Qwen3.5-397B-A17B" --trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 --ep-size $EP_SIZE \ ---cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ ---mem-fraction-static $MEM_FRAC_STATIC --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ ---context-length $CONTEXT_LENGTH --disable-radix-cache \ ---attention-backend trtllm_mha --moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---tokenizer-worker-num 6 --stream-interval 30 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ -> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_b300.sh b/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_b300.sh deleted file mode 100755 index 33a1532554..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_b300.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -export PYTHONUNBUFFERED=1 - -SERVER_LOG=/workspace/server.log - -# Default: recv every ~10 requests; if CONC ≥ 16, relax to ~30 requests between scheduler recv polls. -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -MEM_FRAC_STATIC=0.82 -CHUNKED_PREFILL_SIZE=32768 -MAX_PREFILL_TOKENS=32768 -CUDA_GRAPH_MAX_BATCH_SIZE=$CONC -MAX_RUNNING_REQUESTS=128 -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --host 0.0.0.0 --port $PORT \ ---served-model-name $MODEL --trust-remote-code \ ---tensor-parallel-size $TP --data-parallel-size 1 --ep-size $EP_SIZE \ ---cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ ---mem-fraction-static $MEM_FRAC_STATIC --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ ---context-length $CONTEXT_LENGTH --disable-radix-cache \ ---attention-backend trtllm_mha --mm-attention-backend triton_attn --moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---tokenizer-worker-num 6 --stream-interval 30 > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_b300_mtp.sh deleted file mode 100755 index c4a5185b2c..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_b300_mtp.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -export PYTHONUNBUFFERED=1 - -SERVER_LOG=/workspace/server.log - -# Default: recv every ~10 requests; if CONC ≥ 16, relax to ~30 requests between scheduler recv polls. -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -MEM_FRAC_STATIC=0.82 -CHUNKED_PREFILL_SIZE=32768 -MAX_PREFILL_TOKENS=32768 -CUDA_GRAPH_MAX_BATCH_SIZE=$CONC -MAX_RUNNING_REQUESTS=128 -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --host 0.0.0.0 --port $PORT \ ---served-model-name $MODEL --trust-remote-code \ ---tensor-parallel-size $TP --data-parallel-size 1 --ep-size $EP_SIZE \ ---cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ ---mem-fraction-static $MEM_FRAC_STATIC --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ ---context-length $CONTEXT_LENGTH --disable-radix-cache \ ---attention-backend trtllm_mha --mm-attention-backend triton_attn --moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---tokenizer-worker-num 6 --stream-interval 30 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ -> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_mi300x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_mi300x.sh deleted file mode 100755 index b803c55514..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_mi300x.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -# following Andy Luo linkedin's recipe https://www.linkedin.com/feed/update/urn:li:activity:7429203734389280768/ -python3 -m sglang.launch_server \ - --attention-backend aiter \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --data-parallel-size 1 \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.75 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_mi325x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_mi325x.sh deleted file mode 100644 index b803c55514..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_mi325x.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -# following Andy Luo linkedin's recipe https://www.linkedin.com/feed/update/urn:li:activity:7429203734389280768/ -python3 -m sglang.launch_server \ - --attention-backend aiter \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --data-parallel-size 1 \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.75 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_mi325x_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_mi325x_mtp.sh deleted file mode 100755 index 88624f7407..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_mi325x_mtp.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - EP_SIZE \ - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -# following Andy Luo linkedin's recipe https://www.linkedin.com/feed/update/urn:li:activity:7429203734389280768/ -python3 -m sglang.launch_server \ - --attention-backend aiter \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --ep-size $EP_SIZE \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --mem-fraction-static 0.75 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - EP_SIZE \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_mi355x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_mi355x.sh deleted file mode 100755 index 590205c62d..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_mi355x.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -python3 -m sglang.launch_server \ - --attention-backend triton \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --ep-size $EP_SIZE \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.8 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_mi355x_mtp.sh deleted file mode 100755 index 72c8abec45..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_bf16_mi355x_mtp.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -python3 -m sglang.launch_server \ - --attention-backend triton \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --ep-size $EP_SIZE \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.8 \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_fp4_rtx6000pro_sglang.sh b/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_fp4_rtx6000pro_sglang.sh deleted file mode 100755 index 736488cf63..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_fp4_rtx6000pro_sglang.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash - -# SM120 has no trtllm-gen kernels, so MoE and NVFP4 GEMMs run on FlashInfer CUTLASS and attention on FlashInfer. -# The node is PCIe-only, so collectives use plain NCCL instead of the custom all-reduce. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi - SERVE_MODEL="$MODEL_PATH" -else - hf download "$MODEL" - SERVE_MODEL="$MODEL" -fi - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -export SGLANG_ENABLE_JIT_DEEPGEMM=false -export PYTHONUNBUFFERED=1 - -SERVER_LOG=/workspace/server.log - -# 96 GiB per GPU: weights take ~56 GiB per rank and CUDA graphs ~7 GiB. At 0.8 the KV pool -# grew to 2.2M tokens and the first 8k prefill OOM'd on activations; 0.7 still leaves ~1M KV -# tokens, and a 2-request prefill chunk bounds the activation peak. -MEM_FRAC_STATIC="0.7" -CHUNKED_PREFILL_SIZE=$((ISL * 2)) -MAX_PREFILL_TOKENS=$((ISL * 2)) -MAX_RUNNING_REQUESTS=128 -CONTEXT_LENGTH=$((ISL + OSL + 20)) - -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -if [[ "$EVAL_ONLY" == "true" ]]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server \ - --model-path "$SERVE_MODEL" \ - --served-model-name "$MODEL" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --trust-remote-code \ - --tensor-parallel-size "$TP" \ - --data-parallel-size 1 \ - --ep-size "$EP_SIZE" \ - --reasoning-parser qwen3 \ - --tool-call-parser qwen3_coder \ - --quantization modelopt_fp4 \ - --fp4-gemm-backend flashinfer_cutlass \ - --moe-runner-backend flashinfer_cutlass \ - --attention-backend flashinfer \ - --kv-cache-dtype fp8_e4m3 \ - --mamba-ssm-dtype bfloat16 \ - --mamba-scheduler-strategy no_buffer \ - --disable-custom-all-reduce \ - --disable-radix-cache \ - --mem-fraction-static "$MEM_FRAC_STATIC" \ - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" \ - --max-prefill-tokens "$MAX_PREFILL_TOKENS" \ - --context-length "$CONTEXT_LENGTH" \ - --cuda-graph-max-bs-decode "$CONC" \ - --max-running-requests "$MAX_RUNNING_REQUESTS" \ - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" \ - --stream-interval 20 > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready \ - --port "$PORT" \ - --server-log "$SERVER_LOG" \ - --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [[ "$RUN_EVAL" == "true" ]]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_fp4_rtx6000pro_sglang_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_fp4_rtx6000pro_sglang_mtp.sh deleted file mode 100755 index 96a8da8ec2..0000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/qwen3.5_fp4_rtx6000pro_sglang_mtp.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env bash - -# SM120 has no trtllm-gen kernels, so MoE and NVFP4 GEMMs run on FlashInfer CUTLASS and attention on FlashInfer. -# The node is PCIe-only, so collectives use plain NCCL instead of the custom all-reduce. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi - SERVE_MODEL="$MODEL_PATH" -else - hf download "$MODEL" - SERVE_MODEL="$MODEL" -fi - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -export SGLANG_ENABLE_JIT_DEEPGEMM=false -export PYTHONUNBUFFERED=1 - -SERVER_LOG=/workspace/server.log - -# SGLang holds back total * (1 - mem-fraction-static) as slack, so the 4.05 GiB per-rank MTP draft head -# must be paid for by raising the fraction: 0.65 and 0.75 left the KV/Mamba pools empty on this 96 GiB SKU, -# 0.85 left only 4.56 GiB free per rank (too thin for a 16k prefill chunk), so 0.80. -MEM_FRAC_STATIC="0.80" -CHUNKED_PREFILL_SIZE=$((ISL * 2)) -MAX_PREFILL_TOKENS=$((ISL * 2)) -# The client never opens more than CONC connections; a larger Mamba state pool strands memory the draft head needs. -MAX_RUNNING_REQUESTS="$CONC" -CONTEXT_LENGTH=$((ISL + OSL + 20)) - -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -if [[ "$EVAL_ONLY" == "true" ]]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server \ - --model-path "$SERVE_MODEL" \ - --served-model-name "$MODEL" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --trust-remote-code \ - --tensor-parallel-size "$TP" \ - --data-parallel-size 1 \ - --ep-size "$EP_SIZE" \ - --reasoning-parser qwen3 \ - --tool-call-parser qwen3_coder \ - --quantization modelopt_fp4 \ - --fp4-gemm-backend flashinfer_cutlass \ - --moe-runner-backend flashinfer_cutlass \ - --attention-backend flashinfer \ - --kv-cache-dtype fp8_e4m3 \ - --mamba-ssm-dtype bfloat16 \ - --mamba-scheduler-strategy no_buffer \ - --disable-custom-all-reduce \ - --disable-radix-cache \ - --mem-fraction-static "$MEM_FRAC_STATIC" \ - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" \ - --max-prefill-tokens "$MAX_PREFILL_TOKENS" \ - --context-length "$CONTEXT_LENGTH" \ - --cuda-graph-max-bs-decode "$CONC" \ - --max-running-requests "$MAX_RUNNING_REQUESTS" \ - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" \ - --stream-interval 20 \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready \ - --port "$PORT" \ - --server-log "$SERVER_LOG" \ - --server-pid "$SERVER_PID" - -# EAGLE-style spec decoding is trained against chat-formatted inputs, so the -# benchmark must send chat prompts or the acceptance rate silently collapses. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template \ - --trust-remote-code - -if [[ "$RUN_EVAL" == "true" ]]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x