test: validate AMD power exporters on MI300X, MI325X, and MI355X

.github/workflows/benchmark-tmpl.yml

@@ -285,6 +285,10 @@ jobs:
       - name: Prepare result-processing Python
         if: ${{ !inputs.eval-only }}
         run: |
+          export UV_CACHE_DIR="$RUNNER_TEMP/uv-cache"
+          export UV_PYTHON_INSTALL_DIR="$RUNNER_TEMP/uv-python"
+          echo "UV_CACHE_DIR=$UV_CACHE_DIR" >> "$GITHUB_ENV"
+          echo "UV_PYTHON_INSTALL_DIR=$UV_PYTHON_INSTALL_DIR" >> "$GITHUB_ENV"
           uv python install --quiet --no-bin 3.12
           result_python=$(uv python find --system 3.12)
           "$result_python" --version

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml

@@ -45,7 +45,7 @@ roles:
   prefill:
     nodes: 1
     workers: 1
-    gpus: 4
+    gpus: 8
     env: &common_environment
       HF_HOME: "/hf_hub_cache"
     # Hugging Face stores hub snapshots under $HF_HOME/hub. Keep the explicit
@@ -75,7 +75,7 @@ roles:
       SGLANG_HEALTH_CHECK_TIMEOUT: "600"
     args: &common_config
       served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8"
-      tensor-parallel-size: 4
+      tensor-parallel-size: 8
       disaggregation-transfer-backend: mori
       disaggregation-ib-device: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7"
       kv-cache-dtype: "fp8_e4m3"
@@ -112,8 +112,14 @@ health_check:
   max_attempts: 720
   interval_seconds: 5
 
+telemetry:
+  enabled: true
+  required: true
+  startup_timeout_seconds: 180
+
 benchmark:
   type: custom
+  concurrencies: [8]
   command: bash /infmax-workspace/benchmarks/multi_node/srt_fixed_sequence.sh
   env:
     ISL: '8192'

benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8/8k1k.yaml

@@ -16,6 +16,10 @@ base:
     tachometer:
       enabled: false
   engine: sglang
+  telemetry:
+    enabled: true
+    required: true
+    startup_timeout_seconds: 180
   roles:
     agg:
       nodes: 1
@@ -39,6 +43,7 @@ base:
         enable-metrics: false
   benchmark:
     type: custom
+    concurrencies: [4]
     command: bash /infmax-workspace/benchmarks/single_node/srt_fixed_sequence.sh
     env:
       MODEL: Qwen/Qwen3.5-397B-A17B-FP8

benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8/8k1k.yaml

@@ -16,6 +16,10 @@ base:
     tachometer:
       enabled: false
   engine: sglang
+  telemetry:
+    enabled: true
+    required: true
+    startup_timeout_seconds: 180
   roles:
     agg:
       nodes: 1
@@ -39,6 +43,7 @@ base:
         enable-metrics: false
   benchmark:
     type: custom
+    concurrencies: [4]
     command: bash /infmax-workspace/benchmarks/single_node/srt_fixed_sequence.sh
     env:
       MODEL: Qwen/Qwen3.5-397B-A17B-FP8

benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8/8k1k.yaml

@@ -16,14 +16,18 @@ base:
     tachometer:
       enabled: false
   engine: sglang
+  telemetry:
+    enabled: true
+    required: true
+    startup_timeout_seconds: 180
   roles:
     agg:
       nodes: 1
       workers: 1
-      gpus: 4
+      gpus: 8
       args:
         attention-backend: aiter
-        tensor-parallel-size: 4
+        tensor-parallel-size: 8
         expert-parallel-size: 1
         trust-remote-code: true
         tokenizer-worker-num: 6
@@ -45,6 +49,7 @@ base:
         SGLANG_USE_AITER: '1'
   benchmark:
     type: custom
+    concurrencies: [4]
     command: bash /infmax-workspace/benchmarks/single_node/srt_fixed_sequence.sh
     env:
       MODEL: Qwen/Qwen3.5-397B-A17B-FP8

benchmarks/single_node/srt_fixed_sequence.sh

@@ -62,3 +62,8 @@ run_benchmark_serving \
     --result-filename "$RESULT_FILENAME" \
     --result-dir "$RESULT_DIR" \
     "${CLIENT_ARGS[@]}"
+
+if [[ -n "${SRT_MEASUREMENT_WINDOW_DIR:-}" ]]; then
+    PYTHONPATH="$INFERENCEX_REPO_ROOT" python3 -m infx.results.power.window \
+        "$RESULT_DIR/$RESULT_FILENAME.json" "$CONC"
+fi

configs/amd-master.yaml

@@ -194,7 +194,7 @@ qwen3.5-fp8-mi355x-sglang:
     - isl: 8192
       osl: 1024
       search-space:
-      - tp: 4
+      - tp: 8
         ep: 1
         conc-start: 4
         conc-end: 256
@@ -312,21 +312,18 @@ qwen3.5-fp8-mi355x-sglang-disagg:
     - isl: 8192
       osl: 1024
       search-space:
-      # The srt-slurm recipe owns the complete c8-c128 sweep in one Slurm
-      # allocation so model initialization is paid once instead of per point.
-      # 1P+1D TP4P+TP8D/EP1 remains the current baseline: TP4 prefill saves
-      # four GPUs without changing the TP8 decode service.
+      # Power smoke: one concurrency and eight GPUs on each P/D node.
       # dp-attn intentionally false: with --enable-dp-attention +
       # --moe-a2a-backend mori, sglang auto-promotes moe_ep_size=tp_size,
       # but is_deepep_class_backend() excludes MoRI, so
       # num_shared_slots stays at the global value (1) and the
       # (num_experts - num_shared_slots) % moe_ep_size assertion in
       # fused_moe_triton/layer.py fires for Qwen3.5 (512 routed + 1 shared).
       - spec-decoding: "none"
-        conc-list: [8, 16, 32, 64, 128]
+        conc-list: [8]
         prefill:
           num-worker: 1
-          tp: 4
+          tp: 8
           ep: 1
           dp-attn: false
           additional-settings:

runners/launch_mi325x-amds.sh

@@ -20,7 +20,7 @@ if [[ "$EXECUTION_PATH" == native-single-node ]]; then
     export SALLOC_TIME_LIMIT=480
     export SRT_SRUN_OPTIONS='{"container-remap-root":"", "container-writable":""}'
     SRT_SQUASH_FILE="/raid/squash/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
-    launch_srt_single_node mi325x-amds
+    launch_srt_single_node mi325x-amds --var GITHUB_WORKSPACE "$GITHUB_WORKSPACE"
     exit $?
 fi
 

runners/srt-slurm/exporters/amd-power.json

@@ -0,0 +1,8 @@
+{
+  "ServerPort": 19500,
+  "CommonConfig": {"MetricsFieldPrefix": ""},
+  "GPUConfig": {
+    "Fields": ["GPU_POWER_USAGE", "GPU_GFX_ACTIVITY"],
+    "Labels": ["GPU_ID", "SERIAL_NUMBER", "GPU_COMPUTE_PARTITION_TYPE", "GPU_MEMORY_PARTITION_TYPE"]
+  }
+}

runners/srt-slurm/mi300x-amd.yaml

@@ -3,8 +3,13 @@ default_time_limit: ${SRT_DEFAULT_TIME_LIMIT}
 gpus_per_node: 8
 network_interface: ''
 visible_devices_env: ROCR_VISIBLE_DEVICES
+default_gpu_exporter:
+  container_image: docker://rocm/device-metrics-exporter:v1.5.2
+  port: 19500
+  power_profile: amd-device-metrics
 srtctl_root: ${SRTCTL_ROOT}
 default_mounts:
+  ${GITHUB_WORKSPACE}/runners/srt-slurm/exporters/amd-power.json: /etc/metrics/config.json
   /dev/kfd: /dev/kfd
   /dev/dri: /dev/dri
 default_sbatch_directives:

runners/srt-slurm/mi325x-amds.yaml

@@ -3,8 +3,13 @@ default_time_limit: ${SRT_DEFAULT_TIME_LIMIT}
 gpus_per_node: 8
 network_interface: ''
 visible_devices_env: ROCR_VISIBLE_DEVICES
+default_gpu_exporter:
+  container_image: docker://rocm/device-metrics-exporter:v1.5.2
+  port: 19500
+  power_profile: amd-device-metrics
 srtctl_root: ${SRTCTL_ROOT}
 default_mounts:
+  ${GITHUB_WORKSPACE}/runners/srt-slurm/exporters/amd-power.json: /etc/metrics/config.json
   /dev/kfd: /dev/kfd
   /dev/dri: /dev/dri
 default_sbatch_directives:

runners/srt-slurm/mi355x-amds.yaml

@@ -10,7 +10,10 @@ srtctl_root: ${SRTCTL_ROOT}
 
 gpus_per_node: 8
 visible_devices_env: ROCR_VISIBLE_DEVICES
-default_gpu_exporter: null
+default_gpu_exporter:
+  container_image: docker://rocm/device-metrics-exporter:v1.5.2
+  port: 19500
+  power_profile: amd-device-metrics
 network_interface: eno0
 
 use_gpus_per_node_directive: true
@@ -26,6 +29,7 @@ model_paths:
   Qwen3.5-397B-A17B-FP8: /it-share/data/Qwen3.5-397B-A17B-FP8
 
 default_mounts:
+  ${GITHUB_WORKSPACE}/runners/srt-slurm/exporters/amd-power.json: /etc/metrics/config.json
   /dev/kfd: /dev/kfd
   /dev/dri: /dev/dri
   # This host directory is already the Hub cache, not HF_HOME.