From 4bcefb7da5353a24ea2e37cfe17de1195210c865 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:26:45 -0700 Subject: [PATCH 1/6] Update DSv4 TRT image for B200/B300 (non-MTP) to feat-deepseek_v4-2dd03e6 Bumps the TensorRT-LLM DeepSeek-V4-Pro image for dsv4-fp4-b200-trt and dsv4-fp4-b300-trt to ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6. The -mtp variants are intentionally left on feat-deepseek_v4-9aa3715. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index baa0b38a1..887f2fc0b 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -1801,7 +1801,7 @@ dsv4-fp4-b200-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, offloading: cpu, conc-list: [64, 128, 256] } dsv4-fp4-b200-trt: - image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-9aa3715 + image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: b200-dsv4 @@ -3053,7 +3053,7 @@ dsv4-fp4-b300-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, offloading: cpu, conc-list: [128, 256, 512] } dsv4-fp4-b300-trt: - image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-9aa3715 + image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: b300 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 263da4cfa..cb59c07eb 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3354,3 +3354,10 @@ description: - "Add MTP speculative-decoding sibling for dsv4-fp4-mi355x-vllm (model: deepseek-ai/DeepSeek-V4-Pro) on vllm/vllm-openai-rocm:v0.22.0, per vllm-project/vllm#43385" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1630 + +- config-keys: + - dsv4-fp4-b200-trt + - dsv4-fp4-b300-trt + description: + - "Update the TensorRT-LLM DeepSeek-V4-Pro image to ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXXX From 6b7558c481d00bdb0d005ccf481347ed8aff2a57 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:27:13 -0700 Subject: [PATCH 2/6] Backfill PR number in changelog pr-link Co-Authored-By: Claude Opus 4.8 (1M context) --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cb59c07eb..28932a990 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3360,4 +3360,4 @@ - dsv4-fp4-b300-trt description: - "Update the TensorRT-LLM DeepSeek-V4-Pro image to ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1636 From f441f9fbeb857f2c3cfd5066e83c71abeeef25e7 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:52:35 -0700 Subject: [PATCH 3/6] Try official TRT-LLM release image 1.3.0rc15.post1 for DSv4 B200/B300 (non-MTP) Swap dsv4-fp4-b200-trt and dsv4-fp4-b300-trt from the custom ghcr.io semianalysis feat/deepseek_v4 build to the official nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc15.post1 to test whether the official RC can serve DeepSeek-V4-Pro. The -mtp variants are unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index 7f1ac486d..a3bfcc5ae 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -1801,7 +1801,7 @@ dsv4-fp4-b200-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, offloading: cpu, conc-list: [64, 128, 256] } dsv4-fp4-b200-trt: - image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6 + image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc15.post1 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: b200-dsv4 @@ -3049,7 +3049,7 @@ dsv4-fp4-b300-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, offloading: cpu, conc-list: [128, 256, 512] } dsv4-fp4-b300-trt: - image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6 + image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc15.post1 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: b300 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8da6e5738..16ef3702f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3372,5 +3372,5 @@ - dsv4-fp4-b200-trt - dsv4-fp4-b300-trt description: - - "Update the TensorRT-LLM DeepSeek-V4-Pro image to ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6" + - "Try the official NVIDIA TensorRT-LLM release image nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc15.post1 for DeepSeek-V4-Pro (replacing the custom feat/deepseek_v4 build) to see whether the official RC can serve DSv4-Pro" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1636 From 4bc5592f8ef18c2ba8952c8f3b9a9020d439a225 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Mon, 1 Jun 2026 23:47:28 -0700 Subject: [PATCH 4/6] Revert to custom feat/deepseek_v4-2dd03e6 image for DSv4 B200/B300 (non-MTP) The official nvcr.io tensorrt-llm/release:1.3.0rc15.post1 loads DSv4-Pro but its DP-attention path deadlocks/crashes under concurrent load (every dpa=true job hung or failed; only pure-TP conc-1 points passed). Revert to the stable custom build until upstream fixes DSv4 + attention-DP (NVIDIA/TensorRT-LLM#13431). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index a3bfcc5ae..7f1ac486d 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -1801,7 +1801,7 @@ dsv4-fp4-b200-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, offloading: cpu, conc-list: [64, 128, 256] } dsv4-fp4-b200-trt: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc15.post1 + image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: b200-dsv4 @@ -3049,7 +3049,7 @@ dsv4-fp4-b300-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, offloading: cpu, conc-list: [128, 256, 512] } dsv4-fp4-b300-trt: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc15.post1 + image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: b300 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 16ef3702f..8da6e5738 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3372,5 +3372,5 @@ - dsv4-fp4-b200-trt - dsv4-fp4-b300-trt description: - - "Try the official NVIDIA TensorRT-LLM release image nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc15.post1 for DeepSeek-V4-Pro (replacing the custom feat/deepseek_v4 build) to see whether the official RC can serve DSv4-Pro" + - "Update the TensorRT-LLM DeepSeek-V4-Pro image to ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1636 From 14a1bb32713c1b24741a987d3460bce6a45d7cda Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Tue, 2 Jun 2026 02:39:16 -0700 Subject: [PATCH 5/6] Point DSv4 B200/B300 TRT (non-MTP) at the SWA-scratch-fix image Bump dsv4-fp4-b200-trt and dsv4-fp4-b300-trt to ghcr.io#semianalysisai/trtllm-deepseek-v4:fix-dsv4-swa-scratch-revert-shrink-c914d6d (TRT-LLM feat/deepseek_v4 @ 084cf2ba + kv_cache_manager_v2 fix). This resolves the engine crash on attention-DP context/generation reverts at high concurrency (the b300 8k1k conc>=512 "LLM is shutting down" hang). The -mtp variants stay on feat-deepseek_v4-9aa3715. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index 87ebd7c2e..8dfac57b5 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -1801,7 +1801,7 @@ dsv4-fp4-b200-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, offloading: cpu, conc-list: [64, 128, 256] } dsv4-fp4-b200-trt: - image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6 + image: ghcr.io#semianalysisai/trtllm-deepseek-v4:fix-dsv4-swa-scratch-revert-shrink-c914d6d model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: b200-dsv4 @@ -3049,7 +3049,7 @@ dsv4-fp4-b300-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, offloading: cpu, conc-list: [128, 256, 512] } dsv4-fp4-b300-trt: - image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6 + image: ghcr.io#semianalysisai/trtllm-deepseek-v4:fix-dsv4-swa-scratch-revert-shrink-c914d6d model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: b300 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ddd6c84fc..ae99f0455 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3372,7 +3372,7 @@ - dsv4-fp4-b200-trt - dsv4-fp4-b300-trt description: - - "Update the TensorRT-LLM DeepSeek-V4-Pro image to ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6" + - "Update the TensorRT-LLM DeepSeek-V4-Pro image to ghcr.io/semianalysisai/trtllm-deepseek-v4:fix-dsv4-swa-scratch-revert-shrink-c914d6d (TRT-LLM feat/deepseek_v4 @ 084cf2ba + kv_cache_manager_v2 fix: free SWA scratch slots on shrink instead of asserting, which crashed the engine on attention-DP context/generation reverts at high concurrency, e.g. b300 8k1k conc>=512)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1636 - config-keys: From e23a5415c387e4c81bde634aaa926d8fdfdc0730 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Tue, 2 Jun 2026 12:32:38 -0700 Subject: [PATCH 6/6] Revert DSv4 B200/B300 TRT (non-MTP) to 2dd03e6 + disable SWA scratch reuse The c914d6d image's kv_cache_manager_v2 patch was wrong: freeing SWA scratch slots on the attention-DP revert->resize(shrink) path hits finish_event=None (a deferred request never forwarded), crashing every dpa=true job and hanging the engine. Root cause is a V2-scheduler / SWA-scratch-reuse conflict: the V2 scheduler grows a context request's KV cache (incl. SWA scratch) before delay batching can defer it, so revert_allocate_context -> resize(shrink) must release scratch slots that have no finish_event. Revert both non-MTP images to feat-deepseek_v4-2dd03e6 and set TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE=0 in the launchers so no scratch slots are allocated and the revert shrinks cleanly. MTP configs untouched (9aa3715). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/configs/nvidia-master.yaml | 4 ++-- benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_trt.sh | 7 +++++++ benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_trt.sh | 7 +++++++ perf-changelog.yaml | 2 +- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index f3837a606..718f95196 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -1801,7 +1801,7 @@ dsv4-fp4-b200-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, offloading: cpu, conc-list: [64, 128, 256] } dsv4-fp4-b200-trt: - image: ghcr.io#semianalysisai/trtllm-deepseek-v4:fix-dsv4-swa-scratch-revert-shrink-c914d6d + image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: b200-dsv4 @@ -3049,7 +3049,7 @@ dsv4-fp4-b300-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, offloading: cpu, conc-list: [128, 256, 512] } dsv4-fp4-b300-trt: - image: ghcr.io#semianalysisai/trtllm-deepseek-v4:fix-dsv4-swa-scratch-revert-shrink-c914d6d + image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: b300 diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_trt.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_trt.sh index e4a24dea2..f2b19a8cb 100644 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_trt.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_trt.sh @@ -47,6 +47,13 @@ sanitize_slurm_mpi_env_for_trtllm export NCCL_NVLS_ENABLE="${NCCL_NVLS_ENABLE:-0}" echo "NCCL_NVLS_ENABLE: $NCCL_NVLS_ENABLE" +# Disable DSv4 SWA scratch reuse: with attention-DP the V2 scheduler grows ctx KV +# (incl. SWA scratch) before delay batching defers a request, and the resulting +# revert_allocate_context -> resize(shrink) can't release the scratch of a +# never-forwarded request (no finish_event), crashing every dpa=true job. +export TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE="${TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE:-0}" +echo "TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE: $TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE" + if [[ "$MODEL" != /* ]]; then hf download "$MODEL" fi diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_trt.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_trt.sh index b0150e10d..3606c677f 100644 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_trt.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_trt.sh @@ -59,6 +59,13 @@ sanitize_slurm_mpi_env_for_trtllm export NCCL_NVLS_ENABLE="${NCCL_NVLS_ENABLE:-0}" echo "NCCL_NVLS_ENABLE: $NCCL_NVLS_ENABLE" +# Disable DSv4 SWA scratch reuse: with attention-DP the V2 scheduler grows ctx KV +# (incl. SWA scratch) before delay batching defers a request, and the resulting +# revert_allocate_context -> resize(shrink) can't release the scratch of a +# never-forwarded request (no finish_event), crashing every dpa=true job. +export TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE="${TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE:-0}" +echo "TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE: $TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE" + nvidia-smi SERVER_LOG="$PWD/server.log" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a6258aee1..17d162518 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3382,7 +3382,7 @@ - dsv4-fp4-b200-trt - dsv4-fp4-b300-trt description: - - "Update the TensorRT-LLM DeepSeek-V4-Pro image to ghcr.io/semianalysisai/trtllm-deepseek-v4:fix-dsv4-swa-scratch-revert-shrink-c914d6d (TRT-LLM feat/deepseek_v4 @ 084cf2ba + kv_cache_manager_v2 fix: free SWA scratch slots on shrink instead of asserting, which crashed the engine on attention-DP context/generation reverts at high concurrency, e.g. b300 8k1k conc>=512)" + - "Revert the non-MTP TensorRT-LLM DeepSeek-V4-Pro image to ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-2dd03e6 and disable DSv4 SWA scratch reuse via TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE=0 in the launcher. Root cause of the prior attention-DP hangs/crashes: the V2 scheduler grows a context request's KV cache (incl. SWA scratch slots) before delay batching can defer it, so revert_allocate_context -> resize(shrink) must release scratch slots of a never-forwarded request, which has no finish_event -> crash on every dpa=true job. Disabling scratch reuse stops those slots from being allocated so the revert shrinks cleanly." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1636 - config-keys: