From 7453c5ebf635ba998a12726cbfa53f873e800519 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 22:26:28 -0500 Subject: [PATCH 1/3] Add MI355X Qwen3.5 FP4 AgentX MTP Add an SGLang EAGLE AgentX recipe, B200/B300-parity TP4 and TP2 discovery coverage, MI355X HiCache tiers, and required server metrics. Use SGLang v0.5.17 and disable unstable AITER all-reduce fusion to preserve TP2/EP2 rank consistency. --- .../agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh | 178 ++++++++++++++++++ configs/amd-master.yaml | 18 +- perf-changelog.yaml | 8 + 3 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh new file mode 100644 index 000000000..b82b86a08 --- /dev/null +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# AgentX trace replay for Qwen3.5-397B-A17B MXFP4 on MI355X with SGLang +# native EAGLE MTP. Throughput uses the committed golden synthetic +# acceptance length; evaluation retains real target-model verification. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +export EVAL_FRAMEWORK="lm-eval" + +check_env_vars \ + MODEL TP CONC EP_SIZE KV_OFFLOADING \ + TOTAL_CPU_DRAM_GB RESULT_DIR DURATION + +SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30} + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + +rocm-smi || true +amd-smi || true + +export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k +resolve_trace_source +install_agentic_deps + +export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" + +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "SGLang server" 60 + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +CACHE_ARGS=() +WARMUP_ARGS=() +PAGE_SIZE=16 +if require_agentic_kv_offload_backend hicache; then + # Qwen3.5 creates target KV and Mamba host pools per rank. NEXTN adds a + # one-layer draft KV pool, or 1/15 of target KV. Convert the node-level + # workflow budget to SGLang's per-rank --hicache-size and retain the + # established MI355X 180 GB per-pool ceiling. + HICACHE_ALIGNMENT_RESERVE_GB=$TP + HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB)) + if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then + echo "Error: insufficient DRAM after HiCache alignment reserve." >&2 + exit 1 + fi + MAX_HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) + HICACHE_MAX_SIZE_GB_PER_RANK_POOL=${HICACHE_MAX_SIZE_GB_PER_RANK_POOL:-180} + if [ "$MAX_HICACHE_SIZE_GB" -gt "$HICACHE_MAX_SIZE_GB_PER_RANK_POOL" ]; then + MAX_HICACHE_SIZE_GB="$HICACHE_MAX_SIZE_GB_PER_RANK_POOL" + fi + HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}" + if [ "$HICACHE_SIZE_GB" -lt 1 ] || [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then + echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB outside 1..$MAX_HICACHE_SIZE_GB." >&2 + exit 1 + fi + PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) + if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then + echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured ${TOTAL_CPU_DRAM_GB} GB." >&2 + exit 1 + fi + echo "HiCache pools: ${HICACHE_SIZE_GB} GB per rank; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB." + + # Offloaded runs use page size 1 with the proven ROCm direct/layer_first + # host-transfer path. + PAGE_SIZE=1 + CACHE_ARGS=( + --enable-hierarchical-cache + --hicache-size "$HICACHE_SIZE_GB" + --hicache-io-backend direct + --hicache-mem-layout layer_first + --hicache-write-policy write_through_selective + ) + WARMUP_ARGS=(--skip-server-warmup) +fi + +PARALLEL_ARGS=( + --tp "$TP" + --dp 1 + --ep-size "$EP_SIZE" +) + +TOKENIZER_ARGS=() +if [ "$TP" -ge 4 ]; then + TOKENIZER_ARGS=(--tokenizer-worker-num 6) +fi + +MAX_RUNNING_REQUESTS=$((2 * CONC)) +CUDA_GRAPH_MAX_BS="$CONC" +[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 +if agentic_kv_offload_enabled && [ "$CUDA_GRAPH_MAX_BS" -gt 16 ]; then + CUDA_GRAPH_MAX_BS=16 +fi + +export PYTHONNOUSERSITE=1 +export SGLANG_USE_AITER=1 +export SGLANG_USE_AITER_UNIFIED_ATTN=1 +export AITER_FLYDSL_FORCE=1 +export SGLANG_MAMBA_SSM_DTYPE=bfloat16 +export SGLANG_TIMEOUT_KEEP_ALIVE=1800 + +if [ "${EVAL_ONLY:-false}" != "true" ]; then + export SGLANG_SIMULATE_ACC_LEN=3.39 + export SGLANG_SIMULATE_ACC_METHOD=match-expected + export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token +fi + +SGLANG_CMD=( + python3 -m sglang.launch_server + --model-path "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + --trust-remote-code + "${PARALLEL_ARGS[@]}" + --attention-backend aiter + --mem-fraction-static 0.80 + --model-loader-extra-config '{"enable_multithread_load": true}' + --watchdog-timeout 1200 + --page-size "$PAGE_SIZE" + --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --max-prefill-tokens 32768 + --chunked-prefill-size 32768 + --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" + --stream-interval 50 + "${TOKENIZER_ARGS[@]}" + --tokenizer-path "$MODEL" + --reasoning-parser qwen3 + --tool-call-parser qwen3_coder + --speculative-algorithm EAGLE + --speculative-num-steps 3 + --speculative-eagle-topk 1 + --speculative-num-draft-tokens 4 + --enable-metrics + --enable-cache-report + "${CACHE_ARGS[@]}" + "${WARMUP_ARGS[@]}" +) + +printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" +printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" +"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "${EVAL_ONLY:-false}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + REPLAY_CMD+=" --apply-chat-template" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 9bc224b16..87e40f745 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -353,6 +353,23 @@ qwen3.5-fp4-mi355x-sglang-mtp: - { tp: 2, conc-start: 4, conc-end: 128, spec-decoding: mtp } - { tp: 4, conc-start: 4, conc-end: 16, spec-decoding: mtp } +qwen3.5-fp4-mi355x-sglang-agentic-mtp: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260811 + model: amd/Qwen3.5-397B-A17B-MXFP4 + model-prefix: qwen3.5 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64, 68, 72] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76] } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32] } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 34, 36, 38, 40, 44, 48, 52, 56] } + qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 model: amd/Qwen3.5-397B-A17B-MXFP4 @@ -1633,4 +1650,3 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: search-space: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12, 16], spec-decoding: mtp } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4], spec-decoding: mtp } - diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0bce8d894..8fe53c47d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5818,3 +5818,11 @@ description: - "Extend the SimpleCPUOffloadConnector grid to c8/c12/c16/c20/c24/c28/c32/c48/c64 to locate its crossover against the resident curve" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + description: + - "Add Qwen3.5-397B-A17B MXFP4 MI355X SGLang AgentX with native EAGLE MTP and the committed golden synthetic acceptance length." + - "Match the B200/B300 TP4 and TP2 frontier coverage, with MI355X HiCache DRAM tiers only around the corresponding HBM capacity knees and required SGLang metrics exports." + - "Use SGLang v0.5.17 and disable unstable AITER all-reduce fusion for TP2/EP2 EAGLE rank consistency." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2562 From 3099f2ead28ae2c33971fb9bd1e9f81dfc46f8ed Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 03:35:45 -0500 Subject: [PATCH 2/3] test: isolate supported Qwen3.5 HiCache path --- .../agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh | 11 ++++++----- configs/amd-master.yaml | 6 ++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh index b82b86a08..ea3daa8db 100644 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh @@ -85,14 +85,15 @@ if require_agentic_kv_offload_backend hicache; then fi echo "HiCache pools: ${HICACHE_SIZE_GB} GB per rank; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB." - # Offloaded runs use page size 1 with the proven ROCm direct/layer_first - # host-transfer path. - PAGE_SIZE=1 + # Qwen3.5's hybrid attention/Mamba pools require the page-first kernel + # transfer path on ROCm. Page size 64 also matches SGLang's MI355X EAGLE + # recipe and avoids page-size-1 verify-graph compiler failures. + PAGE_SIZE=64 CACHE_ARGS=( --enable-hierarchical-cache --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend direct - --hicache-mem-layout layer_first + --hicache-io-backend kernel + --hicache-mem-layout page_first --hicache-write-policy write_through_selective ) WARMUP_ARGS=(--skip-server-warmup) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 87e40f745..441ce9a0d 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -365,10 +365,8 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64, 68, 72] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76] } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32] } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 34, 36, 38, 40, 44, 48, 52, 56] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64] } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32] } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 From ba9a413eb86aacb35045a25b157d45d13ed92708 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 04:07:19 -0500 Subject: [PATCH 3/3] finalize MI355X Qwen3.5 AgentX sweep --- .../agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh | 55 +------------------ configs/amd-master.yaml | 4 +- perf-changelog.yaml | 2 +- 3 files changed, 5 insertions(+), 56 deletions(-) diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh index ea3daa8db..804d65b08 100644 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh @@ -11,8 +11,7 @@ source "$(dirname "$0")/../../benchmark_lib.sh" export EVAL_FRAMEWORK="lm-eval" check_env_vars \ - MODEL TP CONC EP_SIZE KV_OFFLOADING \ - TOTAL_CPU_DRAM_GB RESULT_DIR DURATION + MODEL TP CONC EP_SIZE RESULT_DIR DURATION SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30} @@ -54,51 +53,6 @@ trap cleanup_agentic_services EXIT trap 'exit 130' INT trap 'exit 143' TERM -CACHE_ARGS=() -WARMUP_ARGS=() -PAGE_SIZE=16 -if require_agentic_kv_offload_backend hicache; then - # Qwen3.5 creates target KV and Mamba host pools per rank. NEXTN adds a - # one-layer draft KV pool, or 1/15 of target KV. Convert the node-level - # workflow budget to SGLang's per-rank --hicache-size and retain the - # established MI355X 180 GB per-pool ceiling. - HICACHE_ALIGNMENT_RESERVE_GB=$TP - HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB)) - if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then - echo "Error: insufficient DRAM after HiCache alignment reserve." >&2 - exit 1 - fi - MAX_HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) - HICACHE_MAX_SIZE_GB_PER_RANK_POOL=${HICACHE_MAX_SIZE_GB_PER_RANK_POOL:-180} - if [ "$MAX_HICACHE_SIZE_GB" -gt "$HICACHE_MAX_SIZE_GB_PER_RANK_POOL" ]; then - MAX_HICACHE_SIZE_GB="$HICACHE_MAX_SIZE_GB_PER_RANK_POOL" - fi - HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}" - if [ "$HICACHE_SIZE_GB" -lt 1 ] || [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then - echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB outside 1..$MAX_HICACHE_SIZE_GB." >&2 - exit 1 - fi - PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) - if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured ${TOTAL_CPU_DRAM_GB} GB." >&2 - exit 1 - fi - echo "HiCache pools: ${HICACHE_SIZE_GB} GB per rank; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB." - - # Qwen3.5's hybrid attention/Mamba pools require the page-first kernel - # transfer path on ROCm. Page size 64 also matches SGLang's MI355X EAGLE - # recipe and avoids page-size-1 verify-graph compiler failures. - PAGE_SIZE=64 - CACHE_ARGS=( - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend kernel - --hicache-mem-layout page_first - --hicache-write-policy write_through_selective - ) - WARMUP_ARGS=(--skip-server-warmup) -fi - PARALLEL_ARGS=( --tp "$TP" --dp 1 @@ -113,9 +67,6 @@ fi MAX_RUNNING_REQUESTS=$((2 * CONC)) CUDA_GRAPH_MAX_BS="$CONC" [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 -if agentic_kv_offload_enabled && [ "$CUDA_GRAPH_MAX_BS" -gt 16 ]; then - CUDA_GRAPH_MAX_BS=16 -fi export PYTHONNOUSERSITE=1 export SGLANG_USE_AITER=1 @@ -142,7 +93,7 @@ SGLANG_CMD=( --mem-fraction-static 0.80 --model-loader-extra-config '{"enable_multithread_load": true}' --watchdog-timeout 1200 - --page-size "$PAGE_SIZE" + --page-size 16 --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" --max-running-requests "$MAX_RUNNING_REQUESTS" --max-prefill-tokens 32768 @@ -159,8 +110,6 @@ SGLANG_CMD=( --speculative-num-draft-tokens 4 --enable-metrics --enable-cache-report - "${CACHE_ARGS[@]}" - "${WARMUP_ARGS[@]}" ) printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 441ce9a0d..4914d4ebb 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -365,8 +365,8 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64] } - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32] } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40] } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8fe53c47d..700053810 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5823,6 +5823,6 @@ - qwen3.5-fp4-mi355x-sglang-agentic-mtp description: - "Add Qwen3.5-397B-A17B MXFP4 MI355X SGLang AgentX with native EAGLE MTP and the committed golden synthetic acceptance length." - - "Match the B200/B300 TP4 and TP2 frontier coverage, with MI355X HiCache DRAM tiers only around the corresponding HBM capacity knees and required SGLang metrics exports." + - "Cover the measured resident TP2/EP2 and TP4 Pareto ranges through their HBM capacity knees, with required SGLang metrics exports." - "Use SGLang v0.5.17 and disable unstable AITER all-reduce fusion for TP2/EP2 EAGLE rank consistency." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2562