From 5b7d30612abf84fc81c5bd1235cabff8f15fbac9 Mon Sep 17 00:00:00 2001 From: Graffioh Date: Tue, 18 Aug 2026 10:41:59 +0000 Subject: [PATCH] fix(qwen35): size mixed concurrent target-step graphs --- server/CMakeLists.txt | 7 +++++ server/src/qwen35/graph_builders.cpp | 5 +++- server/src/qwen35/graph_sizing.h | 33 +++++++++++++++++++++ server/test/test_qwen35_graph_sizing.cpp | 37 ++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 server/src/qwen35/graph_sizing.h create mode 100644 server/test/test_qwen35_graph_sizing.cpp diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 5f0cd92bb..cb8eb625b 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -1117,6 +1117,13 @@ if(DFLASH27B_TESTS) target_link_libraries(test_qwen35_roctx PRIVATE ${CMAKE_DL_LIBS}) list(APPEND _raw_unit_test_targets test_qwen35_roctx) endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_qwen35_graph_sizing.cpp") + add_executable(test_qwen35_graph_sizing + test/test_qwen35_graph_sizing.cpp) + target_include_directories(test_qwen35_graph_sizing PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src) + list(APPEND _raw_unit_test_targets test_qwen35_graph_sizing) + endif() if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_ggml_rmsnorm_batch.cpp") add_executable(test_ggml_rmsnorm_batch test/test_ggml_rmsnorm_batch.cpp) target_include_directories(test_ggml_rmsnorm_batch PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS}) diff --git a/server/src/qwen35/graph_builders.cpp b/server/src/qwen35/graph_builders.cpp index ed22072c8..f47b0f6f0 100644 --- a/server/src/qwen35/graph_builders.cpp +++ b/server/src/qwen35/graph_builders.cpp @@ -4,6 +4,7 @@ #include "delta_net_specla.h" #include "ggml-alloc.h" +#include "graph_sizing.h" #include #include @@ -495,7 +496,9 @@ bool build_target_step( ggml_set_input(sg.logits_row_indices); } - sg.gf = ggml_new_graph_custom(sg.ctx, 16384, false); + const size_t graph_capacity = qwen35_target_graph_capacity( + n_prefill_segments, compact_slots); + sg.gf = ggml_new_graph_custom(sg.ctx, graph_capacity, false); // Step-invariant KV write: only when topology can't vary per step. // DFLASH_QWEN35_NO_KVPAD=1 restores the legacy cpy append + exact-length diff --git a/server/src/qwen35/graph_sizing.h b/server/src/qwen35/graph_sizing.h new file mode 100644 index 000000000..0c22ba86e --- /dev/null +++ b/server/src/qwen35/graph_sizing.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +namespace dflash::common { + +// The historical 16K target graph holds up to eight independent recurrent +// segments. Packed mixed prefill adds one more segment for compact decode: +// on Qwen3.6-27B, eight prefills build 15,061 nodes, while adding decode builds +// 16,741 and overflows the old limit. Each additional segment contributes +// 1,680 nodes (35 nodes in each of 48 DeltaNet layers), so reserve 2K per +// segment beyond the proven eight-segment baseline. This keeps today's common +// shapes at the old capacity while scaling linearly for K=16/32 and beyond. +inline size_t qwen35_target_graph_capacity( + int n_prefill_segments, bool compact_decode) { + constexpr size_t kBaseCapacity = 16384; + constexpr int kBaseRecurrentSegments = 8; + constexpr size_t kGrowthPerSegment = 2048; + + const int prefill_segments = n_prefill_segments > 0 + ? n_prefill_segments + : 0; + const int recurrent_segments = + prefill_segments + (compact_decode ? 1 : 0); + if (recurrent_segments <= kBaseRecurrentSegments) { + return kBaseCapacity; + } + return kBaseCapacity + + (size_t)(recurrent_segments - kBaseRecurrentSegments) * + kGrowthPerSegment; +} + +} // namespace dflash::common diff --git a/server/test/test_qwen35_graph_sizing.cpp b/server/test/test_qwen35_graph_sizing.cpp new file mode 100644 index 000000000..184330500 --- /dev/null +++ b/server/test/test_qwen35_graph_sizing.cpp @@ -0,0 +1,37 @@ +#include "qwen35/graph_sizing.h" + +#include + +using namespace dflash::common; + +namespace { +int failures = 0; + +#define CHECK(condition) do { if (!(condition)) { ++failures; \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #condition); \ +} } while (0) +} // namespace + +int main() { + constexpr size_t old_capacity = 16384; + constexpr size_t measured_eight_prefill_nodes = 15061; + constexpr size_t measured_eight_prefill_decode_nodes = 16741; + + CHECK(qwen35_target_graph_capacity(8, false) == old_capacity); + CHECK(measured_eight_prefill_nodes < old_capacity); + + // C16 regression: eight prompts can prefill while earlier prompts decode. + // The compact decode recurrence is the ninth independent segment. + const size_t mixed_c16_capacity = + qwen35_target_graph_capacity(8, true); + CHECK(measured_eight_prefill_decode_nodes > old_capacity); + CHECK(mixed_c16_capacity == 18432); + CHECK(mixed_c16_capacity > measured_eight_prefill_decode_nodes); + CHECK(mixed_c16_capacity - measured_eight_prefill_decode_nodes == 1691); + + // Preserve linear headroom when the prefill cohort itself grows. + CHECK(qwen35_target_graph_capacity(16, true) == 34816); + CHECK(qwen35_target_graph_capacity(32, true) == 67584); + + return failures == 0 ? 0 : 1; +}