From 760f9ec750c4780694678b47141e7713e4e77107 Mon Sep 17 00:00:00 2001 From: Guanbao Yu Date: Thu, 24 Sep 2026 07:43:31 +0000 Subject: [PATCH 1/3] feat(agentx): bump Kimi-K3 FP4 MI355X ATOM image to 0924 and track recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track recipes/Agentic-Kimi-K3.md as retuned in ROCm/ATOM#2382: enable FlyDSL FP8 prefill attention on every band, and hold a ready prefill for four decode passes from concurrency 16 up. The published concurrency set and every other launch argument are unchanged. 将 MI355X Kimi-K3 FP4 ATOM AgentX 提交切换到 0924 镜像，并跟随 ROCm/ATOM#2382 重调后的 recipe：全部并发开启 FlyDSL FP8 prefill attention；并发 16 及以上时让就绪的 prefill 等待 4 个 decode 轮次。 已发布的并发点集合与其余启动参数保持不变。 Co-Authored-By: Claude Opus 5 --- .../single_node/agentic/kimik3_fp4_mi355x_atom_mtp.sh | 9 +++++++++ configs/amd-master.yaml | 2 +- perf-changelog.yaml | 11 +++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_atom_mtp.sh index 764e40e461..13155407d2 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_atom_mtp.sh @@ -129,6 +129,13 @@ esac export ATOM_ENABLE_REPLAYSSM export AITER_REUSE_IDENTICAL_COMM_GROUPS +# From concurrency 16 up, hold a ready prefill for 4 decode passes instead of +# interleaving it into every step; MAX_QUEUE_MS bounds the wait. 14 runs without it. +if [ "$CONC" -ge 16 ]; then + export ATOM_PREFILL_DECODE_INTERVAL=4 + export ATOM_PREFILL_DELAYER_MAX_QUEUE_MS=5000 +fi + # Full CUDA graphs over [1 .. window * (1 + draft tokens)]: the verify step # submits one row per draft token on top of the accepted token, so capturing # only up to the window would send every speculative decode down the eager path. @@ -199,6 +206,8 @@ export AITER_FLYDSL_STAGE2_FP8=1 # costs more in evictions than its reuse is worth on these traces. export ATOM_STATE_CHECKPOINT_DEMAND=0 export ATOM_GDN_SSM_DTYPE=fp16 +# FlyDSL FP8 prefill attention; ATOM defaults it off. +export ATOM_USE_FLYDSL_FP8_PREFILL_ATTN=1 # https://github.com/SemiAnalysisAI/InferenceX/blob/main/golden_al_distribution/kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml # 7 draft tokens -> AL 3.84 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 421d55c483..0aec4a55d1 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -614,7 +614,7 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: # wider in-flight window needs the deeper pool to keep the paged KV # resident. kimik3-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:ubuntu24.04_py3.12_pytorch_release_2.10.0_kimi_k3_agentic_0911 + image: rocm/atom-dev:ubuntu24.04_py3.12_pytorch_release_2.10.0_kimi_k3_agentic_0924 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5655eb4067..d261cb5689 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8815,3 +8815,14 @@ - Retain min(64*concurrency, 1024) SWA prefix tails within the measured GB200 KV budget; select prefill/decode interval 16 only at TP4 C16 after canonical performance and full GSM8K qualification. - "Match the published vLLM grid exactly: TP2/EP1 and TP4/EP1, each C1/2/4/8/16/32/64/128. Use stock DSpark precision; TP2 uses static memory 0.92, chunk 2048, graph/running cap 16, interval 16, expandable allocator and min(128*CONC,1024) SWA tails. Requalify all 16 performance and full GSM8K points. Avoid deadline-only failures with the supported 12-hour partition maximum for C64/C128 performance while preserving complete warmup and 3600-second scoring." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 + +- config-keys: + - kimik3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Move the MI355X Kimi-K3 FP4 ATOM AgentX submission to image rocm/atom-dev:ubuntu24.04_py3.12_pytorch_release_2.10.0_kimi_k3_agentic_0924, tracking recipes/Agentic-Kimi-K3.md as retuned in ROCm/ATOM#2382." + - "Enable FlyDSL FP8 prefill attention (ATOM_USE_FLYDSL_FP8_PREFILL_ATTN=1) on every concurrency; ATOM defaults it off." + - "Hold a ready prefill for four decode passes from concurrency 16 up (ATOM_PREFILL_DECODE_INTERVAL=4, ATOM_PREFILL_DELAYER_MAX_QUEUE_MS=5000). Concurrency 1, 4 and 14 are unchanged." + - "Keep the published concurrency set [1, 4, 14, 16, 48, 56, 72], max-num-seqs, max-num-batched-tokens, gpu-memory-utilization, the CUDA-graph ladder, dcp-size, draft depth, synthetic acceptance, ReplaySSM placement and LMCache sizing unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 7625dc8fc144bd33625ab00461872bda3249cfd2 Mon Sep 17 00:00:00 2001 From: Guanbao Yu Date: Thu, 24 Sep 2026 07:44:28 +0000 Subject: [PATCH 2/3] chore(agentx): point perf-changelog entry at PR 3407 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 perf-changelog 条目的 pr-link 指向 PR 3407。 Co-Authored-By: Claude Opus 5 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d261cb5689..d0de586d27 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8825,4 +8825,4 @@ - "Enable FlyDSL FP8 prefill attention (ATOM_USE_FLYDSL_FP8_PREFILL_ATTN=1) on every concurrency; ATOM defaults it off." - "Hold a ready prefill for four decode passes from concurrency 16 up (ATOM_PREFILL_DECODE_INTERVAL=4, ATOM_PREFILL_DELAYER_MAX_QUEUE_MS=5000). Concurrency 1, 4 and 14 are unchanged." - "Keep the published concurrency set [1, 4, 14, 16, 48, 56, 72], max-num-seqs, max-num-batched-tokens, gpu-memory-utilization, the CUDA-graph ladder, dcp-size, draft depth, synthetic acceptance, ReplaySSM placement and LMCache sizing unchanged." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3407 From d9c84f1cf81e15de8cff1fd48e9e53da420fd736 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 25 Sep 2026 19:29:15 -0700 Subject: [PATCH 3/3] chore(agentx): bump Kimi-K3 FP4 MI355X ATOM image to nightly_202609251613 Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index da783085e1..c771f07db5 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -690,7 +690,7 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: # wider in-flight window needs the deeper pool to keep the paged KV # resident. kimik3-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:ubuntu24.04_py3.12_pytorch_release_2.10.0_kimi_k3_agentic_0924 + image: rocm/atom-dev:nightly_202609251613 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:mi355x-amds