From 2771c1650142033b5a59d2003262c380130e2e62 Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Fri, 7 Aug 2026 04:34:05 +0530 Subject: [PATCH 01/15] perf(dflash): add safe exact prefill bands --- server/docs/DS4.md | 1 + server/docs/ENVIRONMENT.md | 1 + server/src/deepseek4/deepseek4_backend.cpp | 97 +++++++++++++++------- server/src/deepseek4/deepseek4_graph.cpp | 5 +- server/src/deepseek4/deepseek4_internal.h | 13 +++ server/tests/test_deepseek4_unit.cpp | 97 ++++++++++++++++++++++ 6 files changed, 185 insertions(+), 29 deletions(-) diff --git a/server/docs/DS4.md b/server/docs/DS4.md index 6ce6cdb98..8509414df 100644 --- a/server/docs/DS4.md +++ b/server/docs/DS4.md @@ -231,6 +231,7 @@ The runtime logs the chosen split with a `[deepseek4-split] auto-split:` banner. |----------|---------| | `DFLASH_DS4_CUDA_LAYERS` | Override the auto-split heuristic and pin the first `N` DeepSeek4 layers to CUDA. The remaining `43 - N` layers run on the Halo shard. | | `DFLASH_DS4_TIMING` | Enable DS4 timing logs for the layer-split parent and target-shard daemon. Useful for profiling prefill/decode breakdowns; leave unset for normal runs. | +| `DFLASH_DS4_EXACT_PREFILL_BANDS` | Opt in to compressor-safe exact prefill bands up to four tokens on supported layer-range paths. Exact attention remains tokenwise. Leave unset for the default single-token path; `--chunk 1` is the hard fallback. | | `DFLASH_DS4_SPEC` / `DFLASH_DS4_DRAFT` | Enable DSpark and select its GGUF. | | `DFLASH_DS4_DRAFT_BACKEND` / `DFLASH_DS4_DRAFT_GPU` | Backend and device for the in-process drafter. | | `DFLASH_DS4_MOE_TP` | Enable routed-expert partitioning. | diff --git a/server/docs/ENVIRONMENT.md b/server/docs/ENVIRONMENT.md index a877c3812..8f64029a3 100644 --- a/server/docs/ENVIRONMENT.md +++ b/server/docs/ENVIRONMENT.md @@ -98,6 +98,7 @@ consolidation of this list into CLI flags is tracked as follow-up work. - `DFLASH_DS4_DRAFT_BACKEND` - deepseek4_backend.cpp - `DFLASH_DS4_DRAFT_GPU` - deepseek4_backend.cpp - `DFLASH_DS4_DSPARK_DEBUG` - deepseek4_graph.cpp +- `DFLASH_DS4_EXACT_PREFILL_BANDS` - deepseek4_backend.cpp - `DFLASH_DS4_FUSED_VERIFY` - deepseek4_dspark_spec.cpp, deepseek4_loader.cpp - `DFLASH_DS4_HOTNESS_CSV` - deepseek4_backend.cpp - `DFLASH_DS4_MOE_TP` - deepseek4_backend.cpp diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index 22142cd70..541b15056 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -559,6 +559,32 @@ static MoeLayerDesc make_ds4_expert_layer_desc(const DeepSeek4Layer & layer) { } // namespace +int deepseek4_prefill_chunk_tokens(PrefillAttentionMode mode, + bool exact_bands_enabled, + bool batch_supported, + int requested_chunk, + int layer_major_cap) { + if (!batch_supported || requested_chunk <= 1 || layer_major_cap <= 1) { + return 1; + } + + const int bounded_chunk = std::max( + 1, std::min(requested_chunk, layer_major_cap)); + if (mode == PrefillAttentionMode::Exact) { + constexpr int kMaxExactBandTokens = 4; + return exact_bands_enabled + ? std::min(bounded_chunk, kMaxExactBandTokens) + : 1; + } + return prefill_attention_mode_is_approximate(mode) ? bounded_chunk : 1; +} + +bool deepseek4_prefill_chunk_needs_logits(bool is_final_chunk, + bool ends_at_snapshot, + bool capture_requires_logits) { + return is_final_chunk || ends_at_snapshot || capture_requires_logits; +} + DeepSeek4Backend::DeepSeek4Backend(const DeepSeek4BackendConfig & cfg) : cfg_(cfg) {} @@ -1316,25 +1342,26 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, // that snapshot plus the current ubatch, and commits only the final SWA // tail. Learned compressor boundaries are emitted inside the same graph. // - // Mixed hot/cold hybrid execution still has single-token HC semantics, so - // retain the reference path there. --chunk 1 is the explicit fallback. + // Mixed hot/cold hybrid execution without the layer-range runtime still + // has single-token HC semantics. --chunk 1 is the explicit fallback for + // every path. const int requested_chunk = cfg_.chunk > 0 ? cfg_.chunk : w_.n_swa; const int n_total = (int)tokens.size(); // Bound the layer-major graph to the topology validated by the prefill // kernels. Smaller tail chunks use the same scheduler or its reference // fallback. const int layer_major_cap = DS4_MAX_LAYER_MAJOR_PREFILL_TOKENS; - // Only sparse prefill has a qualified batched mixed-owner HC path. Dense - // hybrid execution remains tokenwise; batching it would skip per-token HC - // post-mixing and corrupt the hidden state. - const bool hybrid_batch_supported = - !moe_hybrid_ || cfg_.prefill_mode == PrefillAttentionMode::Sparse; - const int chunk = - !prefill_attention_mode_is_approximate(cfg_.prefill_mode) || - !hybrid_batch_supported - ? 1 - : std::max(1, std::min(requested_chunk, - layer_major_cap)); + const bool layer_range_hybrid = + moe_hybrid_ && (expert_runtime_.compute || expert_backend_); + const bool batch_supported = + !moe_hybrid_ || + cfg_.prefill_mode == PrefillAttentionMode::Sparse || + (cfg_.prefill_mode == PrefillAttentionMode::Exact && + layer_range_hybrid); + const int chunk = deepseek4_prefill_chunk_tokens( + cfg_.prefill_mode, + env_flag_enabled("DFLASH_DS4_EXACT_PREFILL_BANDS"), + batch_supported, requested_chunk, layer_major_cap); int pos = kv_offset; const bool save_snapshot = snap_slot >= 0 && snap_slot < PREFIX_SLOTS && @@ -1421,18 +1448,30 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, DeepSeek4StepTelemetry step_tel; if (timing) step_tel.embed_us = elapsed_us(embed_t0, Clock::now()); + const bool capture_final = i + n_tok > spec_final_from; + const bool capture_snapshot = + !snapshot_saved && i < spec_snap_to && + i + n_tok > spec_snap_from; + const bool capture_requested = + spec_enabled_ && spec_drafter_ && + (capture_final || capture_snapshot); + // The current q=1 hybrid capture path shares the native fused graph, + // whose contract includes a logits sink. Preserve that graph until a + // capture-only variant is independently qualified. + const bool capture_requires_logits = capture_requested && n_tok == 1; + const bool ends_at_snapshot = + save_snapshot && !snapshot_saved && pos + n_tok == snap_pos; + const bool need_logits = deepseek4_prefill_chunk_needs_logits( + i + n_tok == n_total, ends_at_snapshot, + capture_requires_logits); std::vector logits; + std::vector * logits_out = need_logits ? &logits : nullptr; bool ok = false; std::vector hc_state; Ds4VerifyHooks spec_hooks; std::vector spec_cap; Ds4VerifyHooks * hp = nullptr; - const bool capture_final = i + n_tok > spec_final_from; - const bool capture_snapshot = - !snapshot_saved && i < spec_snap_to && - i + n_tok > spec_snap_from; - if (spec_enabled_ && spec_drafter_ && - (capture_final || capture_snapshot)) { + if (capture_requested) { spec_hooks.capture_layer_ids = &spec_drafter_->capture_layer_ids; spec_hooks.capture_out = &spec_cap; int capture_begin = n_tok; @@ -1450,13 +1489,14 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, } spec_hooks.capture_token_begin = capture_begin; spec_hooks.capture_token_end = capture_end; + spec_hooks.allow_fused_verify = false; hp = &spec_hooks; } if (moe_hybrid_ && (expert_runtime_.compute || expert_backend_)) { ok = deepseek4_step_layer_range( backend_, cfg_.device.gpu, w_, cache_, hc_state, embed.data(), n_tok, pos, - 0, w_.n_layer, &logits, + 0, w_.n_layer, logits_out, tokens.data() + i, timing ? &step_tel : nullptr, /*allow_decode_graph_reuse=*/true, hp, @@ -1472,12 +1512,11 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, hp, expert_runtime_.compute ? &expert_runtime_ : nullptr); } else { - ok = deepseek4_step_layer_range(backend_, cfg_.device.gpu, w_, cache_, hc_state, - embed.data(), n_tok, pos, - 0, w_.n_layer, &logits, - tokens.data() + i, - timing ? &step_tel : nullptr, - cfg_.prefill_mode != PrefillAttentionMode::Sparse, hp); + ok = deepseek4_step_layer_range( + backend_, cfg_.device.gpu, w_, cache_, hc_state, + embed.data(), n_tok, pos, 0, w_.n_layer, logits_out, + tokens.data() + i, timing ? &step_tel : nullptr, + cfg_.prefill_mode != PrefillAttentionMode::Sparse, hp); } if (ok && hp && !spec_cap.empty()) { const int feat_row = spec_drafter_->n_target_layers * w_.n_embd; @@ -1501,9 +1540,11 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, add_step_tel(tel_acc, step_tel); steps++; } - last_logits_ = std::move(logits); pos += n_tok; - last_logits_pos_ = cache_.cur_pos; + if (need_logits) { + last_logits_ = std::move(logits); + last_logits_pos_ = cache_.cur_pos; + } i += n_tok; if (save_snapshot && !snapshot_saved && pos == snap_pos) { snapshot_saved = snapshot_save(snap_slot); diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index 88a8d8263..a7d9c0434 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -6627,6 +6627,7 @@ bool deepseek4_step_layer_range( const bool fused_verify_candidate = (!moe_hybrid || fused_hybrid_ready) && n_tokens >= 2 && n_tokens <= 4 && verify_hooks && + verify_hooks->allow_fused_verify && layer_begin == 0 && is_last_shard && out_logits && ds4_backend_is_gpu(backend) && ds4_fused_verify_enabled(); const bool heterogeneous_sparse_prefill = @@ -6698,6 +6699,7 @@ bool deepseek4_step_layer_range( chunk_hooks.capture_layer_ids = verify_hooks->capture_layer_ids; chunk_hooks.capture_out = verify_hooks->capture_out ? &chunk_capture : nullptr; chunk_hooks.all_logits_out = verify_hooks->all_logits_out ? &chunk_logits : nullptr; + chunk_hooks.allow_fused_verify = verify_hooks->allow_fused_verify; chunk_hooks_ptr = &chunk_hooks; } if (!deepseek4_step_layer_range( @@ -6860,7 +6862,8 @@ bool deepseek4_step_layer_range( (fused_hybrid_decode && !verify_hooks) ? &fused_hybrid_decode_hooks : verify_hooks; if ((!moe_hybrid || fused_hybrid_ready) && - ((n_tokens >= 2 && n_tokens <= 4 && verify_hooks) || + ((n_tokens >= 2 && n_tokens <= 4 && verify_hooks && + verify_hooks->allow_fused_verify) || fused_hybrid_decode) && layer_begin == 0 && is_last_shard && out_logits && ds4_backend_is_gpu(backend) && ds4_fused_verify_enabled()) { diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index b0e80ec00..4ff874d2b 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -352,6 +352,16 @@ int deepseek4_safe_compressor_batch_tokens(const DeepSeek4Weights & w, int kv_start, int n_tokens); +int deepseek4_prefill_chunk_tokens(PrefillAttentionMode mode, + bool exact_bands_enabled, + bool batch_supported, + int requested_chunk, + int layer_major_cap); + +bool deepseek4_prefill_chunk_needs_logits(bool is_final_chunk, + bool ends_at_snapshot, + bool capture_requires_logits); + // Forward: single step (prefill chunk or decode token). // embed: [n_embd, n_tokens] input embeddings (post-embedding lookup). // hc_state: [n_hc * n_embd] persistent HC residual (updated in-place). @@ -389,6 +399,9 @@ struct Ds4VerifyHooks { std::vector * all_logits_out = nullptr; // [n_vocab * n_tokens] std::vector * argmax_out = nullptr; // [n_tokens], optional GPU result bool prefer_argmax_only = false; // skip logits D2H when available + // Prefill uses the capture fields too, but must never enter the + // intentionally approximate fused-verification path. + bool allow_fused_verify = true; }; bool deepseek4_step_layer_range( diff --git a/server/tests/test_deepseek4_unit.cpp b/server/tests/test_deepseek4_unit.cpp index 8649c37fc..a689f981c 100644 --- a/server/tests/test_deepseek4_unit.cpp +++ b/server/tests/test_deepseek4_unit.cpp @@ -1627,6 +1627,99 @@ static void test_safe_compressor_batch_tokens() { std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } +static void test_exact_prefill_chunk_policy() { + std::fprintf(stderr, " test_exact_prefill_chunk_policy ..."); + + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Exact, /*exact_bands_enabled=*/false, + /*batch_supported=*/true, /*requested_chunk=*/4, + /*layer_major_cap=*/512) == 1); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Exact, /*exact_bands_enabled=*/true, + /*batch_supported=*/true, /*requested_chunk=*/1, + /*layer_major_cap=*/512) == 1); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Exact, true, true, 2, 512) == 2); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Exact, true, true, 3, 512) == 3); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Exact, true, true, 4, 512) == 4); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Exact, true, true, 8, 512) == 4); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Exact, true, + /*batch_supported=*/false, 4, 512) == 1); + + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Dense, false, + /*batch_supported=*/false, 8, 512) == 1); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Sparse, false, + /*batch_supported=*/true, 8, 512) == 8); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Sparse, false, + /*batch_supported=*/true, 1024, 512) == 512); + + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); +} + +static void test_exact_prefill_band_schedule() { + std::fprintf(stderr, " test_exact_prefill_band_schedule ..."); + DeepSeek4Weights w; + w.compress_ratios = {4, 128}; + + for (int width = 1; width <= 4; ++width) { + for (int start = 0; start <= 5; ++start) { + for (int total = 1; total <= 9; ++total) { + int consumed = 0; + while (consumed < total) { + int outer = std::min(width, total - consumed); + int inner = 0; + while (inner < outer) { + const int pos = start + consumed + inner; + const int step = deepseek4_safe_compressor_batch_tokens( + w, pos, outer - inner); + TEST_ASSERT(step >= 1 && step <= width); + TEST_ASSERT((pos % 4) + step <= 4); + inner += step; + } + TEST_ASSERT(inner == outer); + consumed += outer; + } + TEST_ASSERT(consumed == total); + } + } + } + + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); +} + +static void test_prefill_chunk_logits_policy() { + std::fprintf(stderr, " test_prefill_chunk_logits_policy ..."); + TEST_ASSERT(!deepseek4_prefill_chunk_needs_logits( + /*is_final_chunk=*/false, /*ends_at_snapshot=*/false, + /*capture_requires_logits=*/false)); + TEST_ASSERT(deepseek4_prefill_chunk_needs_logits( + /*is_final_chunk=*/true, /*ends_at_snapshot=*/false, + /*capture_requires_logits=*/false)); + TEST_ASSERT(deepseek4_prefill_chunk_needs_logits( + /*is_final_chunk=*/false, /*ends_at_snapshot=*/true, + /*capture_requires_logits=*/false)); + TEST_ASSERT(deepseek4_prefill_chunk_needs_logits( + /*is_final_chunk=*/false, /*ends_at_snapshot=*/false, + /*capture_requires_logits=*/true)); + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); +} + +static void test_prefill_capture_disables_fused_verify() { + std::fprintf(stderr, " test_prefill_capture_disables_fused_verify ..."); + Ds4VerifyHooks verifier_hooks; + TEST_ASSERT(verifier_hooks.allow_fused_verify); + verifier_hooks.allow_fused_verify = false; + TEST_ASSERT(!verifier_hooks.allow_fused_verify); + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); +} + static void test_dspark_park_all_releases_drafter() { std::fprintf(stderr, " test_dspark_park_all_releases_drafter ..."); @@ -3681,6 +3774,10 @@ int main() { test_dspark_loader_contract_and_bounds(backend); test_dspark_confidence_uses_separate_hidden(backend); test_safe_compressor_batch_tokens(); + test_exact_prefill_chunk_policy(); + test_exact_prefill_band_schedule(); + test_prefill_chunk_logits_policy(); + test_prefill_capture_disables_fused_verify(); test_dspark_park_all_releases_drafter(); test_dspark_raw_ring_rollback_after_wrap(backend); test_snapshot_save_restore(); From ed8991707d25cfde956e2d599f4570514bc0146c Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Fri, 7 Aug 2026 04:47:24 +0530 Subject: [PATCH 02/15] fix(dflash): preserve existing prefill schedulers --- server/src/deepseek4/deepseek4_backend.cpp | 19 +++++++++++++------ server/src/deepseek4/deepseek4_internal.h | 3 ++- server/tests/test_deepseek4_unit.cpp | 16 ++++++++++++---- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index 541b15056..e7b7ab9a9 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -581,8 +581,10 @@ int deepseek4_prefill_chunk_tokens(PrefillAttentionMode mode, bool deepseek4_prefill_chunk_needs_logits(bool is_final_chunk, bool ends_at_snapshot, - bool capture_requires_logits) { - return is_final_chunk || ends_at_snapshot || capture_requires_logits; + bool capture_requires_logits, + bool execution_requires_logits) { + return is_final_chunk || ends_at_snapshot || capture_requires_logits || + execution_requires_logits; } DeepSeek4Backend::DeepSeek4Backend(const DeepSeek4BackendConfig & cfg) @@ -1455,15 +1457,20 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, const bool capture_requested = spec_enabled_ && spec_drafter_ && (capture_final || capture_snapshot); - // The current q=1 hybrid capture path shares the native fused graph, - // whose contract includes a logits sink. Preserve that graph until a - // capture-only variant is independently qualified. + // Existing q=1 fused and approximate wide schedulers use the logits + // sink as an execution-path selector. Preserve those original + // topologies: the q=1 per-layer fallback is documented as numerically + // distinct, while sparse/layer-major prefill own their batching and + // capture contracts. Readout elision is therefore limited to the new + // q=2..4 exact bands until sink selection is decoupled from readback. const bool capture_requires_logits = capture_requested && n_tok == 1; + const bool execution_requires_logits = + n_tok == 1 || cfg_.prefill_mode != PrefillAttentionMode::Exact; const bool ends_at_snapshot = save_snapshot && !snapshot_saved && pos + n_tok == snap_pos; const bool need_logits = deepseek4_prefill_chunk_needs_logits( i + n_tok == n_total, ends_at_snapshot, - capture_requires_logits); + capture_requires_logits, execution_requires_logits); std::vector logits; std::vector * logits_out = need_logits ? &logits : nullptr; bool ok = false; diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index 4ff874d2b..1533489a6 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -360,7 +360,8 @@ int deepseek4_prefill_chunk_tokens(PrefillAttentionMode mode, bool deepseek4_prefill_chunk_needs_logits(bool is_final_chunk, bool ends_at_snapshot, - bool capture_requires_logits); + bool capture_requires_logits, + bool execution_requires_logits); // Forward: single step (prefill chunk or decode token). // embed: [n_embd, n_tokens] input embeddings (post-embedding lookup). diff --git a/server/tests/test_deepseek4_unit.cpp b/server/tests/test_deepseek4_unit.cpp index a689f981c..3845f63f9 100644 --- a/server/tests/test_deepseek4_unit.cpp +++ b/server/tests/test_deepseek4_unit.cpp @@ -1698,16 +1698,24 @@ static void test_prefill_chunk_logits_policy() { std::fprintf(stderr, " test_prefill_chunk_logits_policy ..."); TEST_ASSERT(!deepseek4_prefill_chunk_needs_logits( /*is_final_chunk=*/false, /*ends_at_snapshot=*/false, - /*capture_requires_logits=*/false)); + /*capture_requires_logits=*/false, + /*execution_requires_logits=*/false)); TEST_ASSERT(deepseek4_prefill_chunk_needs_logits( /*is_final_chunk=*/true, /*ends_at_snapshot=*/false, - /*capture_requires_logits=*/false)); + /*capture_requires_logits=*/false, + /*execution_requires_logits=*/false)); TEST_ASSERT(deepseek4_prefill_chunk_needs_logits( /*is_final_chunk=*/false, /*ends_at_snapshot=*/true, - /*capture_requires_logits=*/false)); + /*capture_requires_logits=*/false, + /*execution_requires_logits=*/false)); TEST_ASSERT(deepseek4_prefill_chunk_needs_logits( /*is_final_chunk=*/false, /*ends_at_snapshot=*/false, - /*capture_requires_logits=*/true)); + /*capture_requires_logits=*/true, + /*execution_requires_logits=*/false)); + TEST_ASSERT(deepseek4_prefill_chunk_needs_logits( + /*is_final_chunk=*/false, /*ends_at_snapshot=*/false, + /*capture_requires_logits=*/false, + /*execution_requires_logits=*/true)); std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } From ce2cbd92e178aaf1a2490ac8814c0f1fcf7041dd Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Sat, 8 Aug 2026 02:26:09 +0530 Subject: [PATCH 03/15] fix(dflash): harden exact prefill lifecycle --- server/src/deepseek4/deepseek4_backend.cpp | 62 ++++++++++++++--- server/src/deepseek4/deepseek4_graph.cpp | 80 +++++++++++++++++----- server/src/deepseek4/deepseek4_internal.h | 22 ++++++ server/tests/test_deepseek4_unit.cpp | 75 ++++++++++++++++++-- 4 files changed, 205 insertions(+), 34 deletions(-) diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index e7b7ab9a9..4d503f9e7 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -26,6 +26,10 @@ namespace dflash::common { +bool deepseek4_env_flag_value_enabled(const char * value) { + return value && value[0] && std::strcmp(value, "0") != 0; +} + namespace { using Clock = std::chrono::steady_clock; @@ -38,8 +42,7 @@ static uint64_t elapsed_us(Clock::time_point start, Clock::time_point end) { } static bool env_flag_enabled(const char * name) { - const char * value = std::getenv(name); - return value && value[0] && std::strcmp(value, "0") != 0; + return deepseek4_env_flag_value_enabled(std::getenv(name)); } static void configure_gfx1151_dspark_mmvq_default(int gpu) { @@ -587,6 +590,29 @@ bool deepseek4_prefill_chunk_needs_logits(bool is_final_chunk, execution_requires_logits; } +void deepseek4_invalidate_prefill_logits_if_skipped( + bool need_logits, + std::vector & last_logits, + int & last_logits_pos) { + if (need_logits) return; + last_logits.clear(); + last_logits_pos = -1; +} + +Ds4VerifyHooks deepseek4_make_prefill_capture_hooks( + const std::vector * capture_layer_ids, + std::vector * capture_out, + int capture_token_begin, + int capture_token_end) { + Ds4VerifyHooks hooks; + hooks.capture_layer_ids = capture_layer_ids; + hooks.capture_out = capture_out; + hooks.capture_token_begin = capture_token_begin; + hooks.capture_token_end = capture_token_end; + hooks.allow_fused_verify = false; + return hooks; +} + DeepSeek4Backend::DeepSeek4Backend(const DeepSeek4BackendConfig & cfg) : cfg_(cfg) {} @@ -1473,14 +1499,17 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, capture_requires_logits, execution_requires_logits); std::vector logits; std::vector * logits_out = need_logits ? &logits : nullptr; + // A snapshot boundary may have left valid logits for an older cache + // position. Invalidate them before a no-readout forward so a partial + // failure cannot expose stale state. + deepseek4_invalidate_prefill_logits_if_skipped( + need_logits, last_logits_, last_logits_pos_); bool ok = false; std::vector hc_state; Ds4VerifyHooks spec_hooks; std::vector spec_cap; Ds4VerifyHooks * hp = nullptr; if (capture_requested) { - spec_hooks.capture_layer_ids = &spec_drafter_->capture_layer_ids; - spec_hooks.capture_out = &spec_cap; int capture_begin = n_tok; int capture_end = 0; if (capture_final) { @@ -1494,9 +1523,9 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, capture_end = std::max( capture_end, std::min(n_tok, spec_snap_to - i)); } - spec_hooks.capture_token_begin = capture_begin; - spec_hooks.capture_token_end = capture_end; - spec_hooks.allow_fused_verify = false; + spec_hooks = deepseek4_make_prefill_capture_hooks( + &spec_drafter_->capture_layer_ids, &spec_cap, + capture_begin, capture_end); hp = &spec_hooks; } if (moe_hybrid_ && (expert_runtime_.compute || expert_backend_)) { @@ -1620,7 +1649,17 @@ bool DeepSeek4Backend::do_decode(int committed, int n_gen, // Get last logits and sample std::vector logits; - if (generated == 0 && !last_logits_.empty()) { + if (generated == 0 && committed > 0) { + if (last_logits_.size() != (size_t) w_.n_vocab || + last_logits_pos_ != committed || + last_logits_pos_ != cache_.cur_pos) { + std::fprintf(stderr, + "[deepseek4] refusing missing or stale prefill logits " + "(logits_pos=%d committed=%d cache_pos=%d size=%zu)\n", + last_logits_pos_, committed, cache_.cur_pos, + last_logits_.size()); + return false; + } logits = last_logits_; } else { std::vector embed(w_.n_embd); @@ -1771,8 +1810,11 @@ GenerateResult DeepSeek4Backend::generate_from_state( const bool sampling_requires_ar = sampler_.needs_logit_processing(); if (spec_enabled_ && spec_drafter_ && req.n_gen > 0 && !req.force_ar_decode && !budget_requires_ar && !sampling_requires_ar) { - if (last_logits_.empty()) { - result.fail(GenerateErrorCode::DecodeFailed, "spec: no prefill logits"); + if (last_logits_.size() != (size_t) w_.n_vocab || + last_logits_pos_ != committed || + last_logits_pos_ != cache_.cur_pos) { + result.fail(GenerateErrorCode::DecodeFailed, + "spec: missing or stale prefill logits"); return result; } int seed = 0; diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index a7d9c0434..988949439 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -3694,12 +3694,13 @@ static void hc_pre_auto_into(float * working, n_embd, n_hc, sinkhorn_iters, hc_eps, flat, mix_scratch, serial_fn); } -static void hc_pre_batch(std::vector & working, +static bool hc_pre_batch(std::vector & working, std::vector & post, std::vector & comb, const float * hc_state, const HcWeightsCpu & weights, ggml_tensor * fn_tensor, + int device, int n_tokens, int n_embd, int n_hc, @@ -3710,7 +3711,20 @@ static void hc_pre_batch(std::vector & working, post.resize((size_t)n_tokens * (size_t)n_hc); comb.resize((size_t)n_tokens * (size_t)n_hc * (size_t)n_hc); + std::atomic device_ready{true}; ds4_pool_for_tokens(n_tokens, [&](int t0, int t1) { +#if defined(DFLASH27B_BACKEND_CUDA) + thread_local int selected_device = -1; + if (selected_device != device) { + if (!deepseek4_cuda_hc_set_device(device)) { + device_ready.store(false, std::memory_order_relaxed); + return; + } + selected_device = device; + } +#else + (void) device; +#endif std::vector flat(hc_dim); float mix[24]; for (int t = t0; t < t1; ++t) { @@ -3729,6 +3743,7 @@ static void hc_pre_batch(std::vector & working, /*serial_fn=*/n_tokens > 1); } }); + return device_ready.load(std::memory_order_relaxed); } static void cpu_hc_post(float * out_hc, const float * block_out, @@ -6585,6 +6600,21 @@ static bool initialize_layer_range_cache( runtime.owns_output = owns_output; return true; } + +bool deepseek4_should_attempt_fused_verify( + int n_tokens, + const Ds4VerifyHooks * verify_hooks, + bool owner_topology_supported, + bool full_layer_range, + bool has_logits_output, + bool gpu_backend, + bool fused_verify_enabled) { + return owner_topology_supported && n_tokens >= 2 && n_tokens <= 4 && + verify_hooks && verify_hooks->allow_fused_verify && + full_layer_range && has_logits_output && gpu_backend && + fused_verify_enabled; +} + bool deepseek4_step_layer_range( ggml_backend_t backend, int device, @@ -6624,12 +6654,13 @@ bool deepseek4_step_layer_range( moe_hybrid->materialized_cold_experts && moe_hybrid->cold_backend_kind == MoeHybridColdBackend::Gpu && moe_hybrid->cold_backend && moe_hybrid->cold_backend != backend; - const bool fused_verify_candidate = - (!moe_hybrid || fused_hybrid_ready) && - n_tokens >= 2 && n_tokens <= 4 && verify_hooks && - verify_hooks->allow_fused_verify && - layer_begin == 0 && is_last_shard && out_logits && - ds4_backend_is_gpu(backend) && ds4_fused_verify_enabled(); + const bool fused_verify_candidate = deepseek4_should_attempt_fused_verify( + n_tokens, verify_hooks, + !moe_hybrid || fused_hybrid_ready, + layer_begin == 0 && is_last_shard, + out_logits != nullptr, + ds4_backend_is_gpu(backend), + ds4_fused_verify_enabled()); const bool heterogeneous_sparse_prefill = moe_hybrid && cache.prefill_mode == PrefillAttentionMode::Sparse && n_tokens > 4 && n_tokens <= DS4_MAX_LAYER_MAJOR_PREFILL_TOKENS && @@ -6861,12 +6892,11 @@ bool deepseek4_step_layer_range( Ds4VerifyHooks * fused_graph_hooks = (fused_hybrid_decode && !verify_hooks) ? &fused_hybrid_decode_hooks : verify_hooks; - if ((!moe_hybrid || fused_hybrid_ready) && - ((n_tokens >= 2 && n_tokens <= 4 && verify_hooks && - verify_hooks->allow_fused_verify) || - fused_hybrid_decode) && - layer_begin == 0 && is_last_shard && - out_logits && ds4_backend_is_gpu(backend) && ds4_fused_verify_enabled()) { + const bool fused_hybrid_decode_candidate = + fused_hybrid_decode && layer_begin == 0 && is_last_shard && + out_logits && ds4_backend_is_gpu(backend) && + ds4_fused_verify_enabled(); + if (fused_verify_candidate || fused_hybrid_decode_candidate) { const bool q1_feature_capture = n_tokens == 1 && verify_hooks && verify_hooks->capture_out; // q=1 target-feature capture walks many prompt-position shapes. Keep @@ -7148,9 +7178,15 @@ bool deepseek4_step_layer_range( attn_post_backend = cached.post; attn_comb_backend = cached.comb; } else { - hc_pre_batch(cur, hc_post, hc_comb, - hc_state.data(), hc_lw.attn, L.hc_attn_fn, - n_tokens, n_embd, n_hc, w.n_hc_sinkhorn_iter, w.hc_eps); + if (!hc_pre_batch(cur, hc_post, hc_comb, + hc_state.data(), hc_lw.attn, L.hc_attn_fn, + device, n_tokens, n_embd, n_hc, + w.n_hc_sinkhorn_iter, w.hc_eps)) { + std::fprintf(stderr, + "[deepseek4] HC-pre device selection failed " + "layer %d attn\n", il); + return false; + } } if (telemetry) telemetry->hc_pre_attn_us += ds4_elapsed_us(hc_pre_attn_t0, Ds4TimingClock::now()); @@ -7506,9 +7542,15 @@ bool deepseek4_step_layer_range( ffn_post_backend = cached.post; ffn_comb_backend = cached.comb; } else { - hc_pre_batch(ffn_working, hc_post, hc_comb, - hc_state.data(), hc_lw.ffn, L.hc_ffn_fn, - n_tokens, n_embd, n_hc, w.n_hc_sinkhorn_iter, w.hc_eps); + if (!hc_pre_batch(ffn_working, hc_post, hc_comb, + hc_state.data(), hc_lw.ffn, L.hc_ffn_fn, + device, n_tokens, n_embd, n_hc, + w.n_hc_sinkhorn_iter, w.hc_eps)) { + std::fprintf(stderr, + "[deepseek4] HC-pre device selection failed " + "layer %d ffn\n", il); + return false; + } } if (telemetry) telemetry->hc_pre_ffn_us += ds4_elapsed_us(hc_pre_ffn_t0, Ds4TimingClock::now()); diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index 1533489a6..fbb2994f9 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -358,11 +358,18 @@ int deepseek4_prefill_chunk_tokens(PrefillAttentionMode mode, int requested_chunk, int layer_major_cap); +bool deepseek4_env_flag_value_enabled(const char * value); + bool deepseek4_prefill_chunk_needs_logits(bool is_final_chunk, bool ends_at_snapshot, bool capture_requires_logits, bool execution_requires_logits); +void deepseek4_invalidate_prefill_logits_if_skipped( + bool need_logits, + std::vector & last_logits, + int & last_logits_pos); + // Forward: single step (prefill chunk or decode token). // embed: [n_embd, n_tokens] input embeddings (post-embedding lookup). // hc_state: [n_hc * n_embd] persistent HC residual (updated in-place). @@ -405,6 +412,21 @@ struct Ds4VerifyHooks { bool allow_fused_verify = true; }; +Ds4VerifyHooks deepseek4_make_prefill_capture_hooks( + const std::vector * capture_layer_ids, + std::vector * capture_out, + int capture_token_begin, + int capture_token_end); + +bool deepseek4_should_attempt_fused_verify( + int n_tokens, + const Ds4VerifyHooks * verify_hooks, + bool owner_topology_supported, + bool full_layer_range, + bool has_logits_output, + bool gpu_backend, + bool fused_verify_enabled); + bool deepseek4_step_layer_range( ggml_backend_t backend, int device, diff --git a/server/tests/test_deepseek4_unit.cpp b/server/tests/test_deepseek4_unit.cpp index 3845f63f9..2b6f5acfd 100644 --- a/server/tests/test_deepseek4_unit.cpp +++ b/server/tests/test_deepseek4_unit.cpp @@ -1413,6 +1413,8 @@ static void test_backend_sampling_penalizes_prompt_history() { auto decode_one = [&](const SamplerCfg & sampler) { backend.last_logits_ = {0.0f, 4.0f, 3.0f}; + backend.last_logits_pos_ = 1; + backend.cache_.cur_pos = 1; backend.sampler_ = sampler; emitted.clear(); std::vector generated; @@ -1434,6 +1436,24 @@ static void test_backend_sampling_penalizes_prompt_history() { penalized.rep_pen = 2.0f; TEST_ASSERT(decode_one(penalized) == 2); + backend.last_logits_ = {0.0f, 4.0f, 3.0f}; + backend.last_logits_pos_ = 0; + backend.cache_.cur_pos = 1; + std::vector generated; + emitted.clear(); + TEST_ASSERT(!backend.do_decode( + /*committed=*/1, /*n_gen=*/1, /*history_prefix=*/{1}, + generated, io)); + TEST_ASSERT(emitted.empty()); + + backend.last_logits_.clear(); + backend.last_logits_pos_ = -1; + generated.clear(); + TEST_ASSERT(!backend.do_decode( + /*committed=*/1, /*n_gen=*/1, /*history_prefix=*/{1}, + generated, io)); + TEST_ASSERT(emitted.empty()); + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } @@ -1630,6 +1650,11 @@ static void test_safe_compressor_batch_tokens() { static void test_exact_prefill_chunk_policy() { std::fprintf(stderr, " test_exact_prefill_chunk_policy ..."); + TEST_ASSERT(!deepseek4_env_flag_value_enabled(nullptr)); + TEST_ASSERT(!deepseek4_env_flag_value_enabled("")); + TEST_ASSERT(!deepseek4_env_flag_value_enabled("0")); + TEST_ASSERT(deepseek4_env_flag_value_enabled("1")); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( PrefillAttentionMode::Exact, /*exact_bands_enabled=*/false, /*batch_supported=*/true, /*requested_chunk=*/4, @@ -1719,12 +1744,52 @@ static void test_prefill_chunk_logits_policy() { std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } -static void test_prefill_capture_disables_fused_verify() { - std::fprintf(stderr, " test_prefill_capture_disables_fused_verify ..."); +static void test_prefill_readout_lifecycle_and_fused_exclusion() { + std::fprintf(stderr, + " test_prefill_readout_lifecycle_and_fused_exclusion ..."); + + std::vector last_logits = {1.0f, 2.0f}; + int last_logits_pos = 7; + deepseek4_invalidate_prefill_logits_if_skipped( + /*need_logits=*/true, last_logits, last_logits_pos); + TEST_ASSERT(last_logits == std::vector({1.0f, 2.0f})); + TEST_ASSERT(last_logits_pos == 7); + deepseek4_invalidate_prefill_logits_if_skipped( + /*need_logits=*/false, last_logits, last_logits_pos); + TEST_ASSERT(last_logits.empty()); + TEST_ASSERT(last_logits_pos == -1); + + std::vector capture_ids = {1, 3}; + std::vector capture; + Ds4VerifyHooks prefill_hooks = deepseek4_make_prefill_capture_hooks( + &capture_ids, &capture, /*capture_token_begin=*/1, + /*capture_token_end=*/3); + TEST_ASSERT(prefill_hooks.capture_layer_ids == &capture_ids); + TEST_ASSERT(prefill_hooks.capture_out == &capture); + TEST_ASSERT(prefill_hooks.capture_token_begin == 1); + TEST_ASSERT(prefill_hooks.capture_token_end == 3); + TEST_ASSERT(!prefill_hooks.allow_fused_verify); + TEST_ASSERT(!deepseek4_should_attempt_fused_verify( + /*n_tokens=*/4, &prefill_hooks, + /*owner_topology_supported=*/true, + /*full_layer_range=*/true, + /*has_logits_output=*/true, + /*gpu_backend=*/true, + /*fused_verify_enabled=*/true)); + Ds4VerifyHooks verifier_hooks; TEST_ASSERT(verifier_hooks.allow_fused_verify); - verifier_hooks.allow_fused_verify = false; - TEST_ASSERT(!verifier_hooks.allow_fused_verify); + TEST_ASSERT(deepseek4_should_attempt_fused_verify( + /*n_tokens=*/4, &verifier_hooks, + /*owner_topology_supported=*/true, + /*full_layer_range=*/true, + /*has_logits_output=*/true, + /*gpu_backend=*/true, + /*fused_verify_enabled=*/true)); + TEST_ASSERT(!deepseek4_should_attempt_fused_verify( + /*n_tokens=*/1, &verifier_hooks, true, true, true, true, true)); + TEST_ASSERT(!deepseek4_should_attempt_fused_verify( + /*n_tokens=*/5, &verifier_hooks, true, true, true, true, true)); std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } @@ -3785,7 +3850,7 @@ int main() { test_exact_prefill_chunk_policy(); test_exact_prefill_band_schedule(); test_prefill_chunk_logits_policy(); - test_prefill_capture_disables_fused_verify(); + test_prefill_readout_lifecycle_and_fused_exclusion(); test_dspark_park_all_releases_drafter(); test_dspark_raw_ring_rollback_after_wrap(backend); test_snapshot_save_restore(); From 4049d9975238959ab38132762b1802c5f8aae7b9 Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Tue, 11 Aug 2026 00:21:41 +0530 Subject: [PATCH 04/15] fix(dflash): separate exact prefill logits readback --- server/src/deepseek4/deepseek4_backend.cpp | 98 +++++++++---- .../src/deepseek4/deepseek4_fused_verify.inc | 16 ++- server/src/deepseek4/deepseek4_graph.cpp | 86 ++++++++---- server/src/deepseek4/deepseek4_internal.h | 31 +++- server/tests/test_deepseek4_unit.cpp | 132 ++++++++++++++---- 5 files changed, 269 insertions(+), 94 deletions(-) diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index 4d503f9e7..aa0c55c47 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -582,23 +582,54 @@ int deepseek4_prefill_chunk_tokens(PrefillAttentionMode mode, return prefill_attention_mode_is_approximate(mode) ? bounded_chunk : 1; } -bool deepseek4_prefill_chunk_needs_logits(bool is_final_chunk, - bool ends_at_snapshot, - bool capture_requires_logits, - bool execution_requires_logits) { - return is_final_chunk || ends_at_snapshot || capture_requires_logits || - execution_requires_logits; +DeepSeek4PrefillOutputIntent deepseek4_prefill_output_intent( + PrefillAttentionMode mode, + bool exact_bands_active, + int n_tokens, + bool is_final_chunk, + bool ends_at_snapshot, + bool external_requires_logits) { + const bool legacy_readback = + !exact_bands_active && + (n_tokens == 1 || mode != PrefillAttentionMode::Exact); + const bool readback_logits = + is_final_chunk || ends_at_snapshot || external_requires_logits || + legacy_readback; + return { + /*execute_output_path=*/ + readback_logits || n_tokens == 1 || + mode != PrefillAttentionMode::Exact, + readback_logits, + }; } void deepseek4_invalidate_prefill_logits_if_skipped( - bool need_logits, + bool readback_logits, std::vector & last_logits, int & last_logits_pos) { - if (need_logits) return; + if (readback_logits) return; last_logits.clear(); last_logits_pos = -1; } +bool deepseek4_commit_prefill_logits( + bool readback_logits, + int vocab_size, + int cache_position, + std::vector && logits, + std::vector & last_logits, + int & last_logits_pos) { + if (!readback_logits) return true; + if (vocab_size <= 0 || logits.size() != (size_t) vocab_size) { + last_logits.clear(); + last_logits_pos = -1; + return false; + } + last_logits = std::move(logits); + last_logits_pos = cache_position; + return true; +} + Ds4VerifyHooks deepseek4_make_prefill_capture_hooks( const std::vector * capture_layer_ids, std::vector * capture_out, @@ -1386,10 +1417,16 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, cfg_.prefill_mode == PrefillAttentionMode::Sparse || (cfg_.prefill_mode == PrefillAttentionMode::Exact && layer_range_hybrid); + const bool exact_bands_enabled = + env_flag_enabled("DFLASH_DS4_EXACT_PREFILL_BANDS"); const int chunk = deepseek4_prefill_chunk_tokens( cfg_.prefill_mode, - env_flag_enabled("DFLASH_DS4_EXACT_PREFILL_BANDS"), + exact_bands_enabled, batch_supported, requested_chunk, layer_major_cap); + const bool exact_bands_active = + cfg_.prefill_mode == PrefillAttentionMode::Exact && + exact_bands_enabled && batch_supported && requested_chunk > 1 && + chunk > 1; int pos = kv_offset; const bool save_snapshot = snap_slot >= 0 && snap_slot < PREFIX_SLOTS && @@ -1483,27 +1520,25 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, const bool capture_requested = spec_enabled_ && spec_drafter_ && (capture_final || capture_snapshot); - // Existing q=1 fused and approximate wide schedulers use the logits - // sink as an execution-path selector. Preserve those original - // topologies: the q=1 per-layer fallback is documented as numerically - // distinct, while sparse/layer-major prefill own their batching and - // capture contracts. Readout elision is therefore limited to the new - // q=2..4 exact bands until sink selection is decoupled from readback. - const bool capture_requires_logits = capture_requested && n_tok == 1; - const bool execution_requires_logits = - n_tok == 1 || cfg_.prefill_mode != PrefillAttentionMode::Exact; const bool ends_at_snapshot = save_snapshot && !snapshot_saved && pos + n_tok == snap_pos; - const bool need_logits = deepseek4_prefill_chunk_needs_logits( - i + n_tok == n_total, ends_at_snapshot, - capture_requires_logits, execution_requires_logits); + // Execution topology and host vocabulary readback are independent. + // An exact-band singleton leaf keeps the established q=1/fused graph, + // but only a final/snapshot/external consumer receives host logits. + // Capture hooks request feature rows, not vocabulary values. + const DeepSeek4PrefillOutputIntent output_intent = + deepseek4_prefill_output_intent( + cfg_.prefill_mode, exact_bands_active, n_tok, + i + n_tok == n_total, ends_at_snapshot, + /*external_requires_logits=*/false); std::vector logits; - std::vector * logits_out = need_logits ? &logits : nullptr; + std::vector * logits_out = + output_intent.readback_logits ? &logits : nullptr; // A snapshot boundary may have left valid logits for an older cache // position. Invalidate them before a no-readout forward so a partial // failure cannot expose stale state. deepseek4_invalidate_prefill_logits_if_skipped( - need_logits, last_logits_, last_logits_pos_); + output_intent.readback_logits, last_logits_, last_logits_pos_); bool ok = false; std::vector hc_state; Ds4VerifyHooks spec_hooks; @@ -1538,7 +1573,7 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, /*allow_decode_graph_reuse=*/true, hp, moe_hybrid_.get(), expert_runtime_.compute ? &expert_runtime_ : nullptr, - routing_stats_.get()); + routing_stats_.get(), output_intent.execute_output_path); } else if (moe_hybrid_) { ok = deepseek4_step(backend_, cfg_.device.gpu, w_, cache_, embed.data(), n_tok, pos, logits, moe_hybrid_.get(), tokens.data() + i, @@ -1552,7 +1587,10 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, backend_, cfg_.device.gpu, w_, cache_, hc_state, embed.data(), n_tok, pos, 0, w_.n_layer, logits_out, tokens.data() + i, timing ? &step_tel : nullptr, - cfg_.prefill_mode != PrefillAttentionMode::Sparse, hp); + cfg_.prefill_mode != PrefillAttentionMode::Sparse, hp, + /*moe_hybrid=*/nullptr, /*expert_runtime=*/nullptr, + /*routing_stats=*/nullptr, + output_intent.execute_output_path); } if (ok && hp && !spec_cap.empty()) { const int feat_row = spec_drafter_->n_target_layers * w_.n_embd; @@ -1577,9 +1615,13 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, steps++; } pos += n_tok; - if (need_logits) { - last_logits_ = std::move(logits); - last_logits_pos_ = cache_.cur_pos; + if (!deepseek4_commit_prefill_logits( + output_intent.readback_logits, w_.n_vocab, cache_.cur_pos, + std::move(logits), last_logits_, last_logits_pos_)) { + std::fprintf(stderr, + "[deepseek4] invalid prefill logits at pos=%d\n", + cache_.cur_pos); + return -1; } i += n_tok; if (save_snapshot && !snapshot_saved && pos == snap_pos) { diff --git a/server/src/deepseek4/deepseek4_fused_verify.inc b/server/src/deepseek4/deepseek4_fused_verify.inc index ac583ad65..0add4896b 100644 --- a/server/src/deepseek4/deepseek4_fused_verify.inc +++ b/server/src/deepseek4/deepseek4_fused_verify.inc @@ -1010,7 +1010,7 @@ static int ds4_try_fused_verify_step( const float * embed, int n_tokens, int kv_start, - std::vector & out_logits, + std::vector * out_logits, const int32_t * token_ids, Ds4VerifyHooks * hooks, DeepSeek4StepTelemetry * telemetry, @@ -1316,13 +1316,13 @@ static int ds4_try_fused_verify_step( ggml_backend_tensor_get(fg->logits, hooks->all_logits_out->data(), 0, sizeof(float) * (size_t) w.n_vocab * q); } - if (!argmax_only) { - out_logits.resize((size_t) w.n_vocab); - ggml_backend_tensor_get(fg->logits, out_logits.data(), + if (!argmax_only && out_logits) { + out_logits->resize((size_t) w.n_vocab); + ggml_backend_tensor_get(fg->logits, out_logits->data(), (size_t) (q - 1) * (size_t) w.n_vocab * sizeof(float), sizeof(float) * (size_t) w.n_vocab); - } else { - out_logits.clear(); + } else if (out_logits) { + out_logits->clear(); } if (hooks->capture_out && ex->capture && ncap > 0) { std::vector flat((size_t) w.n_embd * ncap * q); @@ -1337,7 +1337,9 @@ static int ds4_try_fused_verify_step( } } } - if (telemetry) { + if (telemetry && + (argmax_only || hooks->all_logits_out || out_logits || + (hooks->capture_out && ex->capture && ncap > 0))) { telemetry->full_graph_read_us += ds4_elapsed_us(read_t0, Ds4TimingClock::now()); } return 1; diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index 988949439..5fa934266 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -5210,8 +5210,9 @@ static bool ds4_build_fused_decode_graph( #include "deepseek4_fused_verify.inc" -// Returns 1 on success (out_logits filled), 0 to fall back to the per-layer -// path, -1 on a hard failure after cache state may have been touched. +// Returns 1 on success, 0 to fall back to the per-layer path, or -1 on a hard +// failure after cache state may have been touched. The output graph always +// executes; a null out_logits suppresses only the host vocabulary readback. static int ds4_try_fused_decode_step( DeepSeek4FusedDecodeCache & fc, ggml_backend_t backend, @@ -5223,7 +5224,7 @@ static int ds4_try_fused_decode_step( std::vector & hash_scratch, const float * embed, int kv_start, - std::vector & out_logits, + std::vector * out_logits, const int32_t * token_ids, DeepSeek4StepTelemetry * telemetry) { if (fc.disabled) return 0; @@ -5400,11 +5401,16 @@ static int ds4_try_fused_decode_step( if (telemetry) telemetry->full_graph_compute_us += ds4_elapsed_us(compute_t0, Ds4TimingClock::now()); // ── Read logits ───────────────────────────────────────────────── - const auto read_t0 = Ds4TimingClock::now(); - out_logits.resize((size_t) w.n_vocab); - ggml_backend_tensor_get(fg->logits, out_logits.data(), 0, - sizeof(float) * (size_t) w.n_vocab); - if (telemetry) telemetry->full_graph_read_us += ds4_elapsed_us(read_t0, Ds4TimingClock::now()); + if (out_logits) { + const auto read_t0 = Ds4TimingClock::now(); + out_logits->resize((size_t) w.n_vocab); + ggml_backend_tensor_get(fg->logits, out_logits->data(), 0, + sizeof(float) * (size_t) w.n_vocab); + if (telemetry) { + telemetry->full_graph_read_us += + ds4_elapsed_us(read_t0, Ds4TimingClock::now()); + } + } return 1; } @@ -6606,12 +6612,12 @@ bool deepseek4_should_attempt_fused_verify( const Ds4VerifyHooks * verify_hooks, bool owner_topology_supported, bool full_layer_range, - bool has_logits_output, + bool execute_output_path, bool gpu_backend, bool fused_verify_enabled) { return owner_topology_supported && n_tokens >= 2 && n_tokens <= 4 && verify_hooks && verify_hooks->allow_fused_verify && - full_layer_range && has_logits_output && gpu_backend && + full_layer_range && execute_output_path && gpu_backend && fused_verify_enabled; } @@ -6633,7 +6639,8 @@ bool deepseek4_step_layer_range( Ds4VerifyHooks * verify_hooks, MoeHybridStorage * moe_hybrid, MoeExpertComputeRuntime * expert_runtime, - MoeHybridRoutingStats * routing_stats) { + MoeHybridRoutingStats * routing_stats, + bool execute_output_path) { const auto step_t0 = Ds4TimingClock::now(); if (!deepseek4_cuda_hc_set_device(device)) { @@ -6648,6 +6655,12 @@ bool deepseek4_step_layer_range( const int n_hc = w.n_hc; const int hc_dim = n_hc * n_embd; const bool is_last_shard = (layer_end >= w.n_layer); + const bool readback_logits = out_logits != nullptr; + const bool external_output_consumer = + verify_hooks && + (verify_hooks->all_logits_out || verify_hooks->argmax_out); + execute_output_path = + execute_output_path || readback_logits || external_output_consumer; const bool fused_hybrid_ready = moe_hybrid && !expert_runtime && @@ -6658,7 +6671,7 @@ bool deepseek4_step_layer_range( n_tokens, verify_hooks, !moe_hybrid || fused_hybrid_ready, layer_begin == 0 && is_last_shard, - out_logits != nullptr, + execute_output_path, ds4_backend_is_gpu(backend), ds4_fused_verify_enabled()); const bool heterogeneous_sparse_prefill = @@ -6733,18 +6746,28 @@ bool deepseek4_step_layer_range( chunk_hooks.allow_fused_verify = verify_hooks->allow_fused_verify; chunk_hooks_ptr = &chunk_hooks; } + const bool final_chunk = off + chunk == n_tokens; + std::vector * chunk_output = nullptr; + if (out_logits && (!is_last_shard || final_chunk)) { + chunk_output = &chunk_out; + } + const bool chunk_execute_output_path = + (cache.prefill_mode == PrefillAttentionMode::Exact && + chunk == 1) || + (final_chunk && execute_output_path); if (!deepseek4_step_layer_range( backend, device, w, cache, chunk_hc, embed + (size_t) off * input_width, chunk, kv_start + off, layer_begin, layer_end, - out_logits ? &chunk_out : nullptr, + chunk_output, token_ids ? token_ids + off : nullptr, telemetry, allow_decode_graph_reuse, chunk_hooks_ptr, - moe_hybrid, expert_runtime, routing_stats)) { + moe_hybrid, expert_runtime, routing_stats, + chunk_execute_output_path)) { return false; } hc_all.insert(hc_all.end(), chunk_hc.begin(), chunk_hc.end()); - if (out_logits) { + if (chunk_output) { if (is_last_shard) { last_out = std::move(chunk_out); } else { @@ -6894,7 +6917,7 @@ bool deepseek4_step_layer_range( ? &fused_hybrid_decode_hooks : verify_hooks; const bool fused_hybrid_decode_candidate = fused_hybrid_decode && layer_begin == 0 && is_last_shard && - out_logits && ds4_backend_is_gpu(backend) && + execute_output_path && ds4_backend_is_gpu(backend) && ds4_fused_verify_enabled(); if (fused_verify_candidate || fused_hybrid_decode_candidate) { const bool q1_feature_capture = @@ -6909,7 +6932,7 @@ bool deepseek4_step_layer_range( graph_cache, q1_feature_capture, fused_decode_graph_cache, backend, w, cache, hc_layer_weights_range, hc_output_weights_range, hash_routing_tables_range, - scratch.hash_expert_ids, embed, n_tokens, kv_start, *out_logits, token_ids, + scratch.hash_expert_ids, embed, n_tokens, kv_start, out_logits, token_ids, fused_graph_hooks, telemetry, fused_hybrid_ready ? moe_hybrid : nullptr, routing_stats); if (vrc < 0) return false; @@ -6940,12 +6963,12 @@ bool deepseek4_step_layer_range( if (!moe_hybrid && n_tokens == 1 && allow_decode_graph_reuse && layer_begin == 0 && is_last_shard && !(verify_hooks && verify_hooks->capture_layer_ids && verify_hooks->capture_out) && - out_logits && ds4_backend_is_gpu(backend) && + execute_output_path && ds4_backend_is_gpu(backend) && ds4_fused_decode_enabled(w)) { const int rc = ds4_try_fused_decode_step( fused_decode_graph_cache, backend, w, cache, hc_layer_weights_range, hc_output_weights_range, hash_routing_tables_range, scratch.hash_expert_ids, - embed, kv_start, *out_logits, token_ids, telemetry); + embed, kv_start, out_logits, token_ids, telemetry); if (rc < 0) return false; if (rc > 0) { const int np = kv_start + 1; @@ -7762,7 +7785,7 @@ bool deepseek4_step_layer_range( } // ── Output: HC pre → norm → lm_head (or return hidden state) ──────── - if (is_last_shard && out_logits) { + if (is_last_shard && execute_output_path) { // Final HC pre for output const auto output_t0 = Ds4TimingClock::now(); std::vector & final_embd = scratch.final_embd; @@ -7788,9 +7811,13 @@ bool deepseek4_step_layer_range( if (ggml_backend_graph_compute(backend, cached_decode_output_graph.sg.gf) != GGML_STATUS_SUCCESS) { return false; } - out_logits->resize((size_t)w.n_vocab); - ggml_backend_tensor_get(cached_decode_output_graph.sg.logits, - out_logits->data(), 0, sizeof(float) * (size_t)w.n_vocab); + if (readback_logits) { + out_logits->resize((size_t)w.n_vocab); + ggml_backend_tensor_get( + cached_decode_output_graph.sg.logits, + out_logits->data(), 0, + sizeof(float) * (size_t)w.n_vocab); + } } else { const size_t ctx_size = 16 * 1024 * 1024; ggml_init_params params{}; @@ -7836,11 +7863,14 @@ bool deepseek4_step_layer_range( return false; } - out_logits->resize((size_t)w.n_vocab); - const size_t logits_offset = last_only ? 0 : - (size_t)(n_tokens - 1) * (size_t)w.n_vocab * sizeof(float); - ggml_backend_tensor_get(logits, out_logits->data(), logits_offset, - sizeof(float) * (size_t)w.n_vocab); + if (readback_logits) { + out_logits->resize((size_t)w.n_vocab); + const size_t logits_offset = last_only ? 0 : + (size_t)(n_tokens - 1) * (size_t)w.n_vocab * sizeof(float); + ggml_backend_tensor_get( + logits, out_logits->data(), logits_offset, + sizeof(float) * (size_t)w.n_vocab); + } if (verify_hooks && verify_hooks->all_logits_out) { verify_hooks->all_logits_out->resize((size_t) w.n_vocab * n_tokens); ggml_backend_tensor_get(logits, verify_hooks->all_logits_out->data(), 0, diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index fbb2994f9..a3a8747da 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -360,13 +360,29 @@ int deepseek4_prefill_chunk_tokens(PrefillAttentionMode mode, bool deepseek4_env_flag_value_enabled(const char * value); -bool deepseek4_prefill_chunk_needs_logits(bool is_final_chunk, - bool ends_at_snapshot, - bool capture_requires_logits, - bool execution_requires_logits); +struct DeepSeek4PrefillOutputIntent { + bool execute_output_path = false; + bool readback_logits = false; +}; + +DeepSeek4PrefillOutputIntent deepseek4_prefill_output_intent( + PrefillAttentionMode mode, + bool exact_bands_active, + int n_tokens, + bool is_final_chunk, + bool ends_at_snapshot, + bool external_requires_logits); void deepseek4_invalidate_prefill_logits_if_skipped( - bool need_logits, + bool readback_logits, + std::vector & last_logits, + int & last_logits_pos); + +bool deepseek4_commit_prefill_logits( + bool readback_logits, + int vocab_size, + int cache_position, + std::vector && logits, std::vector & last_logits, int & last_logits_pos); @@ -423,7 +439,7 @@ bool deepseek4_should_attempt_fused_verify( const Ds4VerifyHooks * verify_hooks, bool owner_topology_supported, bool full_layer_range, - bool has_logits_output, + bool execute_output_path, bool gpu_backend, bool fused_verify_enabled); @@ -445,7 +461,8 @@ bool deepseek4_step_layer_range( Ds4VerifyHooks * verify_hooks = nullptr, MoeHybridStorage * moe_hybrid = nullptr, MoeExpertComputeRuntime * expert_runtime = nullptr, - MoeHybridRoutingStats * routing_stats = nullptr); + MoeHybridRoutingStats * routing_stats = nullptr, + bool execute_output_path = false); bool build_deepseek4_moe_hybrid_storage_from_file( const std::string & path, diff --git a/server/tests/test_deepseek4_unit.cpp b/server/tests/test_deepseek4_unit.cpp index 2b6f5acfd..277b0dfd3 100644 --- a/server/tests/test_deepseek4_unit.cpp +++ b/server/tests/test_deepseek4_unit.cpp @@ -1719,28 +1719,89 @@ static void test_exact_prefill_band_schedule() { std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } -static void test_prefill_chunk_logits_policy() { - std::fprintf(stderr, " test_prefill_chunk_logits_policy ..."); - TEST_ASSERT(!deepseek4_prefill_chunk_needs_logits( +static void test_prefill_output_intents() { + std::fprintf(stderr, " test_prefill_output_intents ..."); + + // Legacy/default exact q1 and explicit --chunk 1 retain both the + // established output topology and their historical host readback. + const auto legacy_q1 = deepseek4_prefill_output_intent( + PrefillAttentionMode::Exact, /*exact_bands_active=*/false, + /*n_tokens=*/1, /*is_final_chunk=*/false, + /*ends_at_snapshot=*/false, /*external_requires_logits=*/false); + TEST_ASSERT(legacy_q1.execute_output_path); + TEST_ASSERT(legacy_q1.readback_logits); + + // Dense and sparse policies are outside exact-band readout elision. + for (PrefillAttentionMode mode : + {PrefillAttentionMode::Dense, PrefillAttentionMode::Sparse}) { + const auto intent = deepseek4_prefill_output_intent( + mode, false, 4, false, false, false); + TEST_ASSERT(intent.execute_output_path); + TEST_ASSERT(intent.readback_logits); + } + + // Interior singleton leaves created by enabled q2/q3/q4 exact prefill + // preserve the q1/fused execution topology without a host logits readback. + DeepSeek4Weights failure_weights; + failure_weights.compress_ratios = {4, 128}; + for (int width : {2, 3, 4}) { + TEST_ASSERT(deepseek4_safe_compressor_batch_tokens( + failure_weights, /*kv_start=*/2763, width) == 1); + const auto fallback = deepseek4_prefill_output_intent( + PrefillAttentionMode::Exact, /*exact_bands_active=*/true, + /*n_tokens=*/1, /*is_final_chunk=*/false, + /*ends_at_snapshot=*/false, + /*external_requires_logits=*/false); + TEST_ASSERT_MSG(fallback.execute_output_path, + "exact singleton must preserve output topology"); + TEST_ASSERT_MSG(!fallback.readback_logits, + "exact singleton must skip host logits readback"); + } + + // A capture-only singleton has the same topology requirement, but capture + // values are not an external vocabulary-logits consumer. + const auto capture_only = deepseek4_prefill_output_intent( + PrefillAttentionMode::Exact, true, 1, false, false, false); + TEST_ASSERT(capture_only.execute_output_path); + TEST_ASSERT(!capture_only.readback_logits); + + // Final and exact snapshot endpoints still transfer current logits. + const auto final_singleton = deepseek4_prefill_output_intent( + PrefillAttentionMode::Exact, true, 1, true, false, false); + TEST_ASSERT(final_singleton.execute_output_path); + TEST_ASSERT(final_singleton.readback_logits); + const auto snapshot_singleton = deepseek4_prefill_output_intent( + PrefillAttentionMode::Exact, true, 1, false, true, false); + TEST_ASSERT(snapshot_singleton.execute_output_path); + TEST_ASSERT(snapshot_singleton.readback_logits); + + // A genuine external vocabulary consumer is independently authoritative. + const auto external = deepseek4_prefill_output_intent( + PrefillAttentionMode::Exact, true, 4, false, false, true); + TEST_ASSERT(external.execute_output_path); + TEST_ASSERT(external.readback_logits); + + // Geometry from the hardware failure: the 128-token capture window begins + // at 2891 - 128 == 2763. A q=2 band beginning at 2762 is split into an + // interior singleton ending at 2763, which must not read back logits. + constexpr int prompt_tokens = 2891; + constexpr int capture_begin = prompt_tokens - 128; + TEST_ASSERT(capture_begin == 2763); + TEST_ASSERT(DeepSeek4Backend::capture_safe_prefill_tokens( + /*token_offset=*/2762, /*requested_tokens=*/2, + /*final_capture_from=*/capture_begin, + /*batch_final_capture=*/false, + /*snapshot_pending=*/false, + /*snapshot_capture_from=*/0, + /*snapshot_capture_to=*/0) == 1); + TEST_ASSERT(deepseek4_safe_compressor_batch_tokens( + failure_weights, capture_begin, /*n_tokens=*/2) == 1); + const auto failure_geometry = deepseek4_prefill_output_intent( + PrefillAttentionMode::Exact, true, 1, /*is_final_chunk=*/false, /*ends_at_snapshot=*/false, - /*capture_requires_logits=*/false, - /*execution_requires_logits=*/false)); - TEST_ASSERT(deepseek4_prefill_chunk_needs_logits( - /*is_final_chunk=*/true, /*ends_at_snapshot=*/false, - /*capture_requires_logits=*/false, - /*execution_requires_logits=*/false)); - TEST_ASSERT(deepseek4_prefill_chunk_needs_logits( - /*is_final_chunk=*/false, /*ends_at_snapshot=*/true, - /*capture_requires_logits=*/false, - /*execution_requires_logits=*/false)); - TEST_ASSERT(deepseek4_prefill_chunk_needs_logits( - /*is_final_chunk=*/false, /*ends_at_snapshot=*/false, - /*capture_requires_logits=*/true, - /*execution_requires_logits=*/false)); - TEST_ASSERT(deepseek4_prefill_chunk_needs_logits( - /*is_final_chunk=*/false, /*ends_at_snapshot=*/false, - /*capture_requires_logits=*/false, - /*execution_requires_logits=*/true)); + /*external_requires_logits=*/false); + TEST_ASSERT(failure_geometry.execute_output_path); + TEST_ASSERT(!failure_geometry.readback_logits); std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } @@ -1751,11 +1812,34 @@ static void test_prefill_readout_lifecycle_and_fused_exclusion() { std::vector last_logits = {1.0f, 2.0f}; int last_logits_pos = 7; deepseek4_invalidate_prefill_logits_if_skipped( - /*need_logits=*/true, last_logits, last_logits_pos); + /*readback_logits=*/true, last_logits, last_logits_pos); TEST_ASSERT(last_logits == std::vector({1.0f, 2.0f})); TEST_ASSERT(last_logits_pos == 7); deepseek4_invalidate_prefill_logits_if_skipped( - /*need_logits=*/false, last_logits, last_logits_pos); + /*readback_logits=*/false, last_logits, last_logits_pos); + TEST_ASSERT(last_logits.empty()); + TEST_ASSERT(last_logits_pos == -1); + + // Final/snapshot readbacks must be current, vocabulary-sized values. + constexpr int vocab_size = 4; + TEST_ASSERT(deepseek4_commit_prefill_logits( + /*readback_logits=*/true, vocab_size, /*cache_position=*/2764, + std::vector{1.0f, 2.0f, 3.0f, 4.0f}, + last_logits, last_logits_pos)); + TEST_ASSERT(last_logits.size() == vocab_size); + TEST_ASSERT(last_logits_pos == 2764); + TEST_ASSERT(deepseek4_commit_prefill_logits( + /*readback_logits=*/true, vocab_size, /*cache_position=*/2891, + std::vector{4.0f, 3.0f, 2.0f, 1.0f}, + last_logits, last_logits_pos)); + TEST_ASSERT(last_logits.size() == vocab_size); + TEST_ASSERT(last_logits.front() == 4.0f); + TEST_ASSERT(last_logits_pos == 2891); + + // A malformed readback cannot leave previously current logits visible. + TEST_ASSERT(!deepseek4_commit_prefill_logits( + /*readback_logits=*/true, vocab_size, /*cache_position=*/3000, + std::vector{9.0f}, last_logits, last_logits_pos)); TEST_ASSERT(last_logits.empty()); TEST_ASSERT(last_logits_pos == -1); @@ -3849,7 +3933,7 @@ int main() { test_safe_compressor_batch_tokens(); test_exact_prefill_chunk_policy(); test_exact_prefill_band_schedule(); - test_prefill_chunk_logits_policy(); + test_prefill_output_intents(); test_prefill_readout_lifecycle_and_fused_exclusion(); test_dspark_park_all_releases_drafter(); test_dspark_raw_ring_rollback_after_wrap(backend); From aebdd44149bb9b7d10bb8335a8fce259779945bd Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Tue, 11 Aug 2026 00:34:20 +0530 Subject: [PATCH 05/15] test(dflash): preserve recursive output contracts --- server/src/deepseek4/deepseek4_backend.cpp | 30 ++++++++++---- server/src/deepseek4/deepseek4_graph.cpp | 48 ++++++++++++++++++---- server/src/deepseek4/deepseek4_internal.h | 30 ++++++++++++++ server/tests/test_deepseek4_unit.cpp | 48 +++++++++++++++++++++- 4 files changed, 138 insertions(+), 18 deletions(-) diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index aa0c55c47..7a99a89b4 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -612,6 +612,26 @@ void deepseek4_invalidate_prefill_logits_if_skipped( last_logits_pos = -1; } +DeepSeek4PrefillOutputIntent deepseek4_prepare_prefill_output_intent( + PrefillAttentionMode mode, + bool exact_bands_active, + int n_tokens, + bool is_final_chunk, + bool ends_at_snapshot, + bool external_requires_logits, + std::vector & last_logits, + int & last_logits_pos) { + const DeepSeek4PrefillOutputIntent intent = + deepseek4_prefill_output_intent( + mode, exact_bands_active, n_tokens, is_final_chunk, + ends_at_snapshot, external_requires_logits); + // This preparation function is the production ordering boundary: stale + // values are invalidated before the caller can enter any forward path. + deepseek4_invalidate_prefill_logits_if_skipped( + intent.readback_logits, last_logits, last_logits_pos); + return intent; +} + bool deepseek4_commit_prefill_logits( bool readback_logits, int vocab_size, @@ -1527,18 +1547,14 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, // but only a final/snapshot/external consumer receives host logits. // Capture hooks request feature rows, not vocabulary values. const DeepSeek4PrefillOutputIntent output_intent = - deepseek4_prefill_output_intent( + deepseek4_prepare_prefill_output_intent( cfg_.prefill_mode, exact_bands_active, n_tok, i + n_tok == n_total, ends_at_snapshot, - /*external_requires_logits=*/false); + /*external_requires_logits=*/false, + last_logits_, last_logits_pos_); std::vector logits; std::vector * logits_out = output_intent.readback_logits ? &logits : nullptr; - // A snapshot boundary may have left valid logits for an older cache - // position. Invalidate them before a no-readout forward so a partial - // failure cannot expose stale state. - deepseek4_invalidate_prefill_logits_if_skipped( - output_intent.readback_logits, last_logits_, last_logits_pos_); bool ok = false; std::vector hc_state; Ds4VerifyHooks spec_hooks; diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index 5fa934266..d8a114297 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -6621,6 +6621,35 @@ bool deepseek4_should_attempt_fused_verify( fused_verify_enabled; } +DeepSeek4RecursiveOutputIntent deepseek4_recursive_output_intent( + PrefillAttentionMode mode, + bool parent_execute_output_path, + bool parent_has_output_storage, + bool is_last_shard, + int chunk_tokens, + bool is_final_chunk) { + const bool exact = mode == PrefillAttentionMode::Exact; + return { + /*execute_output_path=*/exact + ? chunk_tokens == 1 || + (is_final_chunk && parent_execute_output_path) + : parent_execute_output_path, + /*pass_output_storage=*/ + parent_has_output_storage && + (!is_last_shard || !exact || is_final_chunk), + }; +} + +bool deepseek4_should_attempt_fused_hybrid_decode( + bool fused_hybrid_decode, + bool full_layer_range, + bool execute_output_path, + bool gpu_backend, + bool fused_verify_enabled) { + return fused_hybrid_decode && full_layer_range && execute_output_path && + gpu_backend && fused_verify_enabled; +} + bool deepseek4_step_layer_range( ggml_backend_t backend, int device, @@ -6747,14 +6776,14 @@ bool deepseek4_step_layer_range( chunk_hooks_ptr = &chunk_hooks; } const bool final_chunk = off + chunk == n_tokens; + const DeepSeek4RecursiveOutputIntent chunk_intent = + deepseek4_recursive_output_intent( + cache.prefill_mode, execute_output_path, + out_logits != nullptr, is_last_shard, chunk, final_chunk); std::vector * chunk_output = nullptr; - if (out_logits && (!is_last_shard || final_chunk)) { + if (chunk_intent.pass_output_storage) { chunk_output = &chunk_out; } - const bool chunk_execute_output_path = - (cache.prefill_mode == PrefillAttentionMode::Exact && - chunk == 1) || - (final_chunk && execute_output_path); if (!deepseek4_step_layer_range( backend, device, w, cache, chunk_hc, embed + (size_t) off * input_width, @@ -6763,7 +6792,7 @@ bool deepseek4_step_layer_range( token_ids ? token_ids + off : nullptr, telemetry, allow_decode_graph_reuse, chunk_hooks_ptr, moe_hybrid, expert_runtime, routing_stats, - chunk_execute_output_path)) { + chunk_intent.execute_output_path)) { return false; } hc_all.insert(hc_all.end(), chunk_hc.begin(), chunk_hc.end()); @@ -6916,9 +6945,10 @@ bool deepseek4_step_layer_range( (fused_hybrid_decode && !verify_hooks) ? &fused_hybrid_decode_hooks : verify_hooks; const bool fused_hybrid_decode_candidate = - fused_hybrid_decode && layer_begin == 0 && is_last_shard && - execute_output_path && ds4_backend_is_gpu(backend) && - ds4_fused_verify_enabled(); + deepseek4_should_attempt_fused_hybrid_decode( + fused_hybrid_decode, layer_begin == 0 && is_last_shard, + execute_output_path, ds4_backend_is_gpu(backend), + ds4_fused_verify_enabled()); if (fused_verify_candidate || fused_hybrid_decode_candidate) { const bool q1_feature_capture = n_tokens == 1 && verify_hooks && verify_hooks->capture_out; diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index a3a8747da..9d8327282 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -373,6 +373,16 @@ DeepSeek4PrefillOutputIntent deepseek4_prefill_output_intent( bool ends_at_snapshot, bool external_requires_logits); +DeepSeek4PrefillOutputIntent deepseek4_prepare_prefill_output_intent( + PrefillAttentionMode mode, + bool exact_bands_active, + int n_tokens, + bool is_final_chunk, + bool ends_at_snapshot, + bool external_requires_logits, + std::vector & last_logits, + int & last_logits_pos); + void deepseek4_invalidate_prefill_logits_if_skipped( bool readback_logits, std::vector & last_logits, @@ -443,6 +453,26 @@ bool deepseek4_should_attempt_fused_verify( bool gpu_backend, bool fused_verify_enabled); +struct DeepSeek4RecursiveOutputIntent { + bool execute_output_path = false; + bool pass_output_storage = false; +}; + +DeepSeek4RecursiveOutputIntent deepseek4_recursive_output_intent( + PrefillAttentionMode mode, + bool parent_execute_output_path, + bool parent_has_output_storage, + bool is_last_shard, + int chunk_tokens, + bool is_final_chunk); + +bool deepseek4_should_attempt_fused_hybrid_decode( + bool fused_hybrid_decode, + bool full_layer_range, + bool execute_output_path, + bool gpu_backend, + bool fused_verify_enabled); + bool deepseek4_step_layer_range( ggml_backend_t backend, int device, diff --git a/server/tests/test_deepseek4_unit.cpp b/server/tests/test_deepseek4_unit.cpp index 277b0dfd3..bbd2fb53e 100644 --- a/server/tests/test_deepseek4_unit.cpp +++ b/server/tests/test_deepseek4_unit.cpp @@ -1815,11 +1815,47 @@ static void test_prefill_readout_lifecycle_and_fused_exclusion() { /*readback_logits=*/true, last_logits, last_logits_pos); TEST_ASSERT(last_logits == std::vector({1.0f, 2.0f})); TEST_ASSERT(last_logits_pos == 7); - deepseek4_invalidate_prefill_logits_if_skipped( - /*readback_logits=*/false, last_logits, last_logits_pos); + const auto prepared_no_readback = deepseek4_prepare_prefill_output_intent( + PrefillAttentionMode::Exact, /*exact_bands_active=*/true, + /*n_tokens=*/1, /*is_final_chunk=*/false, + /*ends_at_snapshot=*/false, /*external_requires_logits=*/false, + last_logits, last_logits_pos); + TEST_ASSERT(prepared_no_readback.execute_output_path); + TEST_ASSERT(!prepared_no_readback.readback_logits); TEST_ASSERT(last_logits.empty()); TEST_ASSERT(last_logits_pos == -1); + // The production recursive selector scopes host-readback suppression to + // exact mode. Dense and sparse retain their legacy per-subchunk output. + for (PrefillAttentionMode mode : + {PrefillAttentionMode::Dense, PrefillAttentionMode::Sparse}) { + for (int width : {1, 2, 3, 4}) { + const auto interior = deepseek4_recursive_output_intent( + mode, /*parent_execute_output_path=*/true, + /*parent_has_output_storage=*/true, + /*is_last_shard=*/true, /*chunk_tokens=*/width, + /*is_final_chunk=*/false); + TEST_ASSERT(interior.execute_output_path); + TEST_ASSERT(interior.pass_output_storage); + } + } + const auto exact_interior = deepseek4_recursive_output_intent( + PrefillAttentionMode::Exact, + /*parent_execute_output_path=*/false, + /*parent_has_output_storage=*/false, + /*is_last_shard=*/true, /*chunk_tokens=*/1, + /*is_final_chunk=*/false); + TEST_ASSERT(exact_interior.execute_output_path); + TEST_ASSERT(!exact_interior.pass_output_storage); + const auto exact_final = deepseek4_recursive_output_intent( + PrefillAttentionMode::Exact, + /*parent_execute_output_path=*/true, + /*parent_has_output_storage=*/true, + /*is_last_shard=*/true, /*chunk_tokens=*/1, + /*is_final_chunk=*/true); + TEST_ASSERT(exact_final.execute_output_path); + TEST_ASSERT(exact_final.pass_output_storage); + // Final/snapshot readbacks must be current, vocabulary-sized values. constexpr int vocab_size = 4; TEST_ASSERT(deepseek4_commit_prefill_logits( @@ -1853,6 +1889,14 @@ static void test_prefill_readout_lifecycle_and_fused_exclusion() { TEST_ASSERT(prefill_hooks.capture_token_begin == 1); TEST_ASSERT(prefill_hooks.capture_token_end == 3); TEST_ASSERT(!prefill_hooks.allow_fused_verify); + TEST_ASSERT(deepseek4_should_attempt_fused_hybrid_decode( + /*fused_hybrid_decode=*/true, /*full_layer_range=*/true, + /*execute_output_path=*/prepared_no_readback.execute_output_path, + /*gpu_backend=*/true, /*fused_verify_enabled=*/true)); + TEST_ASSERT(!deepseek4_should_attempt_fused_hybrid_decode( + /*fused_hybrid_decode=*/true, /*full_layer_range=*/true, + /*execute_output_path=*/false, + /*gpu_backend=*/true, /*fused_verify_enabled=*/true)); TEST_ASSERT(!deepseek4_should_attempt_fused_verify( /*n_tokens=*/4, &prefill_hooks, /*owner_topology_supported=*/true, From 5c0745e2571f77461b1de27625cc2c252a49a9a4 Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Tue, 11 Aug 2026 02:33:30 +0530 Subject: [PATCH 06/15] fix(dflash): preserve exact raw-ring reduction order --- server/src/deepseek4/deepseek4_graph.cpp | 27 ++++++++++++++++++++++- server/src/deepseek4/deepseek4_internal.h | 3 +++ server/tests/test_deepseek4_unit.cpp | 6 +++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index d8a114297..ef00d3b7a 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -196,6 +196,12 @@ int deepseek4_previous_raw_ring_spans( return count; } +bool deepseek4_exact_tokenwise_uses_runtime_raw_row( + int token_position, + int n_swa) { + return n_swa > 0 && token_position >= n_swa - 1; +} + struct DeepSeek4I32InputBinding { ggml_tensor * tensor = nullptr; int32_t value = 0; @@ -5726,11 +5732,25 @@ static bool ds4_run_exact_tokenwise_prefill_attention( std::vector i32_array_inputs; std::vector i64_array_inputs; std::vector f32_array_inputs; + // q=1 switches to a stable physical-ring reduction once the SWA + // window fills. Exact q=2..4 must use the same runtime row topology; + // reconstructing chronological spans changes the F32 reduction order + // after wrap and can move learned routing weights past the oracle + // tolerance even though the raw/compressed cache contents are equal. + DeepSeek4AttentionGraphInputs stable_inputs{}; + const int token_position = kv_start + ti; + if (deepseek4_exact_tokenwise_uses_runtime_raw_row( + token_position, w.n_swa)) { + stable_inputs.raw_kv_rows = + ggml_new_tensor_2d(ctx, GGML_TYPE_I64, 1, 1); + ggml_set_input(stable_inputs.raw_kv_rows); + } ggml_cgraph * gf = ggml_new_graph_custom( ctx, ds4_attn_step_graph_size(1), false); ggml_tensor * normed = build_rms_norm(ctx, inp, L.attn_norm, w.rms_eps); ggml_tensor * attn_out = build_mla_attention( - ctx, gf, normed, w, L, lc, il, kv_start + ti, 1, nullptr, + ctx, gf, normed, w, L, lc, il, token_position, 1, + stable_inputs.raw_kv_rows ? &stable_inputs : nullptr, i32_inputs, i32_array_inputs, i64_array_inputs, &f32_array_inputs, attention_impl); ggml_set_output(attn_out); @@ -5757,6 +5777,11 @@ static bool ds4_run_exact_tokenwise_prefill_attention( ggml_backend_tensor_set(inp, cur + (size_t) ti * n_embd, 0, sizeof(float) * (size_t) n_embd); + if (stable_inputs.raw_kv_rows) { + const int64_t raw_row = token_position % w.n_swa; + ggml_backend_tensor_set( + stable_inputs.raw_kv_rows, &raw_row, 0, sizeof(raw_row)); + } for (const auto & b : i32_inputs) { ggml_backend_tensor_set(b.tensor, &b.value, 0, sizeof(b.value)); } diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index 9d8327282..35fbf92db 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -339,6 +339,9 @@ int deepseek4_previous_raw_ring_spans( int kv_start, int n_swa, DeepSeek4RawRingSpan spans[2]); +bool deepseek4_exact_tokenwise_uses_runtime_raw_row( + int token_position, + int n_swa); bool deepseek4_snapshot_save(const DeepSeek4Cache & cache, ggml_backend_t snapshot_backend, DeepSeek4Snapshot & out); diff --git a/server/tests/test_deepseek4_unit.cpp b/server/tests/test_deepseek4_unit.cpp index bbd2fb53e..8ffb36f7f 100644 --- a/server/tests/test_deepseek4_unit.cpp +++ b/server/tests/test_deepseek4_unit.cpp @@ -1123,6 +1123,12 @@ static void test_raw_ring_spans_after_wrap() { TEST_ASSERT(spans[1].count == 2); TEST_ASSERT(spans[0].count + spans[1].count == 7); + TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(-1, 8)); + TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(6, 8)); + TEST_ASSERT(deepseek4_exact_tokenwise_uses_runtime_raw_row(7, 8)); + TEST_ASSERT(deepseek4_exact_tokenwise_uses_runtime_raw_row(8, 8)); + TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(8, 0)); + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } From de8428c7b672416254fac8ff4e1569952c27aba7 Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Tue, 11 Aug 2026 02:55:37 +0530 Subject: [PATCH 07/15] fix(dflash): scope raw-ring order to exact prefill --- server/src/deepseek4/deepseek4_backend.cpp | 6 ++++-- server/src/deepseek4/deepseek4_graph.cpp | 16 +++++++++++----- server/src/deepseek4/deepseek4_internal.h | 4 +++- server/tests/test_deepseek4_unit.cpp | 14 +++++++++----- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index 7a99a89b4..eb4dce651 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -1589,7 +1589,8 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, /*allow_decode_graph_reuse=*/true, hp, moe_hybrid_.get(), expert_runtime_.compute ? &expert_runtime_ : nullptr, - routing_stats_.get(), output_intent.execute_output_path); + routing_stats_.get(), output_intent.execute_output_path, + exact_bands_active); } else if (moe_hybrid_) { ok = deepseek4_step(backend_, cfg_.device.gpu, w_, cache_, embed.data(), n_tok, pos, logits, moe_hybrid_.get(), tokens.data() + i, @@ -1606,7 +1607,8 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, cfg_.prefill_mode != PrefillAttentionMode::Sparse, hp, /*moe_hybrid=*/nullptr, /*expert_runtime=*/nullptr, /*routing_stats=*/nullptr, - output_intent.execute_output_path); + output_intent.execute_output_path, + exact_bands_active); } if (ok && hp && !spec_cap.empty()) { const int feat_row = spec_drafter_->n_target_layers * w_.n_embd; diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index ef00d3b7a..4175173d6 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -197,9 +197,11 @@ int deepseek4_previous_raw_ring_spans( } bool deepseek4_exact_tokenwise_uses_runtime_raw_row( + bool exact_prefill_stable_raw_order, int token_position, int n_swa) { - return n_swa > 0 && token_position >= n_swa - 1; + return exact_prefill_stable_raw_order && n_swa > 0 && + token_position >= n_swa - 1; } struct DeepSeek4I32InputBinding { @@ -5711,6 +5713,7 @@ static bool ds4_run_exact_tokenwise_prefill_attention( int n_tokens, int kv_start, DeepSeek4AttentionImpl attention_impl, + bool exact_prefill_stable_raw_order, std::vector & attn_out_host, DeepSeek4CachedLayerAlloc & attn_alloc, DeepSeek4StepTelemetry * telemetry) { @@ -5740,7 +5743,7 @@ static bool ds4_run_exact_tokenwise_prefill_attention( DeepSeek4AttentionGraphInputs stable_inputs{}; const int token_position = kv_start + ti; if (deepseek4_exact_tokenwise_uses_runtime_raw_row( - token_position, w.n_swa)) { + exact_prefill_stable_raw_order, token_position, w.n_swa)) { stable_inputs.raw_kv_rows = ggml_new_tensor_2d(ctx, GGML_TYPE_I64, 1, 1); ggml_set_input(stable_inputs.raw_kv_rows); @@ -6694,7 +6697,8 @@ bool deepseek4_step_layer_range( MoeHybridStorage * moe_hybrid, MoeExpertComputeRuntime * expert_runtime, MoeHybridRoutingStats * routing_stats, - bool execute_output_path) { + bool execute_output_path, + bool exact_prefill_stable_raw_order) { const auto step_t0 = Ds4TimingClock::now(); if (!deepseek4_cuda_hc_set_device(device)) { @@ -6817,7 +6821,8 @@ bool deepseek4_step_layer_range( token_ids ? token_ids + off : nullptr, telemetry, allow_decode_graph_reuse, chunk_hooks_ptr, moe_hybrid, expert_runtime, routing_stats, - chunk_intent.execute_output_path)) { + chunk_intent.execute_output_path, + exact_prefill_stable_raw_order)) { return false; } hc_all.insert(hc_all.end(), chunk_hc.begin(), chunk_hc.end()); @@ -7287,7 +7292,8 @@ bool deepseek4_step_layer_range( : DeepSeek4AttentionImpl::Explicit; if (!ds4_run_exact_tokenwise_prefill_attention( backend, w, L, lc, il, cur.data(), n_tokens, kv_start, - attention_impl, attn_out_host, + attention_impl, exact_prefill_stable_raw_order, + attn_out_host, cached_attn_allocs[(size_t) il], telemetry)) { return false; } diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index 35fbf92db..fafd5247b 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -340,6 +340,7 @@ int deepseek4_previous_raw_ring_spans( int n_swa, DeepSeek4RawRingSpan spans[2]); bool deepseek4_exact_tokenwise_uses_runtime_raw_row( + bool exact_prefill_stable_raw_order, int token_position, int n_swa); bool deepseek4_snapshot_save(const DeepSeek4Cache & cache, @@ -495,7 +496,8 @@ bool deepseek4_step_layer_range( MoeHybridStorage * moe_hybrid = nullptr, MoeExpertComputeRuntime * expert_runtime = nullptr, MoeHybridRoutingStats * routing_stats = nullptr, - bool execute_output_path = false); + bool execute_output_path = false, + bool exact_prefill_stable_raw_order = false); bool build_deepseek4_moe_hybrid_storage_from_file( const std::string & path, diff --git a/server/tests/test_deepseek4_unit.cpp b/server/tests/test_deepseek4_unit.cpp index 8ffb36f7f..63f5dd461 100644 --- a/server/tests/test_deepseek4_unit.cpp +++ b/server/tests/test_deepseek4_unit.cpp @@ -1123,11 +1123,15 @@ static void test_raw_ring_spans_after_wrap() { TEST_ASSERT(spans[1].count == 2); TEST_ASSERT(spans[0].count + spans[1].count == 7); - TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(-1, 8)); - TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(6, 8)); - TEST_ASSERT(deepseek4_exact_tokenwise_uses_runtime_raw_row(7, 8)); - TEST_ASSERT(deepseek4_exact_tokenwise_uses_runtime_raw_row(8, 8)); - TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(8, 0)); + TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(true, -1, 8)); + TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(true, 6, 8)); + TEST_ASSERT(deepseek4_exact_tokenwise_uses_runtime_raw_row(true, 7, 8)); + TEST_ASSERT(deepseek4_exact_tokenwise_uses_runtime_raw_row(true, 8, 8)); + TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(true, 8, 0)); + // Default dynamic speculative verification is exact multi-token work too, + // but it must preserve its historical chronological-span topology. + TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(false, 7, 8)); + TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(false, 8, 8)); std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } From c6d5815161e1694ed8abec43563f51c48b3fc5f1 Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Tue, 11 Aug 2026 14:02:10 +0530 Subject: [PATCH 08/15] fix(dflash): preserve q1 hybrid FFN order for exact q4 --- server/src/deepseek4/deepseek4_backend.cpp | 4 +- server/src/deepseek4/deepseek4_graph.cpp | 69 ++++++++++++++++++---- server/src/deepseek4/deepseek4_internal.h | 11 +++- server/tests/test_deepseek4_unit.cpp | 13 ++++ 4 files changed, 82 insertions(+), 15 deletions(-) diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index eb4dce651..6271502e2 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -1590,7 +1590,7 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, moe_hybrid_.get(), expert_runtime_.compute ? &expert_runtime_ : nullptr, routing_stats_.get(), output_intent.execute_output_path, - exact_bands_active); + exact_bands_active, exact_bands_active); } else if (moe_hybrid_) { ok = deepseek4_step(backend_, cfg_.device.gpu, w_, cache_, embed.data(), n_tok, pos, logits, moe_hybrid_.get(), tokens.data() + i, @@ -1608,7 +1608,7 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, /*moe_hybrid=*/nullptr, /*expert_runtime=*/nullptr, /*routing_stats=*/nullptr, output_intent.execute_output_path, - exact_bands_active); + exact_bands_active, exact_bands_active); } if (ok && hp && !spec_cap.empty()) { const int feat_row = spec_drafter_->n_target_layers * w_.n_embd; diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index 4175173d6..c90ed4467 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -204,6 +204,15 @@ bool deepseek4_exact_tokenwise_uses_runtime_raw_row( token_position >= n_swa - 1; } +int deepseek4_exact_prefill_hybrid_ffn_sub_batch( + bool exact_prefill_q1_ffn_order, + int n_tokens) { + // q=1..3 use the same reduced-stack MMVQ reduction order on gfx1151. + // Exactly four rows select a different kernel topology, so retain q=4 + // prompt geometry while evaluating its independent FFN rows as q=3+q=1. + return exact_prefill_q1_ffn_order && n_tokens == 4 ? 3 : n_tokens; +} + struct DeepSeek4I32InputBinding { ggml_tensor * tensor = nullptr; int32_t value = 0; @@ -5437,6 +5446,7 @@ static bool eval_ds4_layer_range_hybrid_ffn( MoeHybridRoutingStats * routing_stats, std::vector & out, DeepSeek4StepTelemetry * telemetry, + bool exact_prefill_q1_ffn_order, const MoeHybridDeviceOutputs * device_outputs = nullptr) { const bool trace_prefill = ds4_env_flag("DFLASH_DS4_PREFILL_TRACE"); if (trace_prefill) { @@ -5679,16 +5689,48 @@ static bool eval_ds4_layer_range_hybrid_ffn( layer); } const auto owners_t0 = Ds4TimingClock::now(); - const bool ok = eval_ds4_hybrid( - backend, hybrid.cpu_backend, cfg, desc, &hybrid, - hybrid.layers[(size_t)layer], nullptr, - layer, n_embd, route_width, - device_ffn_input ? nullptr : normed_host.data(), - selected.data(), weights.data(), - n_tokens, out, hot_alloc, cold_alloc, - expert_compute, expert_layer, telemetry, - device_ffn_input ? normed : nullptr, - device_ffn_input ? device_outputs : nullptr); + const int owner_sub_batch = + deepseek4_exact_prefill_hybrid_ffn_sub_batch( + exact_prefill_q1_ffn_order, n_tokens); + bool ok = true; + if (owner_sub_batch > 0 && owner_sub_batch < n_tokens) { + out.resize((size_t)n_embd * (size_t)n_tokens); + std::vector sub_out; + for (int token_begin = 0; token_begin < n_tokens; + token_begin += owner_sub_batch) { + const int token_count = + std::min(owner_sub_batch, n_tokens - token_begin); + ok = eval_ds4_hybrid( + backend, hybrid.cpu_backend, cfg, desc, &hybrid, + hybrid.layers[(size_t)layer], nullptr, + layer, n_embd, route_width, + normed_host.data() + (size_t)token_begin * (size_t)n_embd, + selected.data() + (size_t)token_begin * (size_t)route_width, + weights.data() + (size_t)token_begin * (size_t)route_width, + token_count, sub_out, /*hot_alloc=*/nullptr, + /*cold_alloc=*/nullptr, expert_compute, expert_layer, + telemetry); + if (!ok || sub_out.size() != + (size_t)n_embd * (size_t)token_count) { + ok = false; + break; + } + std::memcpy( + out.data() + (size_t)token_begin * (size_t)n_embd, + sub_out.data(), sizeof(float) * sub_out.size()); + } + } else { + ok = eval_ds4_hybrid( + backend, hybrid.cpu_backend, cfg, desc, &hybrid, + hybrid.layers[(size_t)layer], nullptr, + layer, n_embd, route_width, + device_ffn_input ? nullptr : normed_host.data(), + selected.data(), weights.data(), + n_tokens, out, hot_alloc, cold_alloc, + expert_compute, expert_layer, telemetry, + device_ffn_input ? normed : nullptr, + device_ffn_input ? device_outputs : nullptr); + } if (trace_prefill) { std::fprintf(stderr, "[deepseek4-prefill-trace] layer=%d expert owners=%s " @@ -6698,7 +6740,8 @@ bool deepseek4_step_layer_range( MoeExpertComputeRuntime * expert_runtime, MoeHybridRoutingStats * routing_stats, bool execute_output_path, - bool exact_prefill_stable_raw_order) { + bool exact_prefill_stable_raw_order, + bool exact_prefill_q1_ffn_order) { const auto step_t0 = Ds4TimingClock::now(); if (!deepseek4_cuda_hc_set_device(device)) { @@ -6822,7 +6865,8 @@ bool deepseek4_step_layer_range( telemetry, allow_decode_graph_reuse, chunk_hooks_ptr, moe_hybrid, expert_runtime, routing_stats, chunk_intent.execute_output_path, - exact_prefill_stable_raw_order)) { + exact_prefill_stable_raw_order, + exact_prefill_q1_ffn_order)) { return false; } hc_all.insert(hc_all.end(), chunk_hc.begin(), chunk_hc.end()); @@ -7680,6 +7724,7 @@ bool deepseek4_step_layer_range( token_ids, hash_routing_tables_range[(size_t)il], *moe_hybrid, expert_runtime, routing_stats, ffn_out_host, telemetry, + exact_prefill_q1_ffn_order, ffn_device_join ? &owner_outputs : nullptr)) { std::fprintf(stderr, "[deepseek4-moe-tp] layer-range FFN failed layer %d\n", diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index fafd5247b..5ef8f2014 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -343,6 +343,14 @@ bool deepseek4_exact_tokenwise_uses_runtime_raw_row( bool exact_prefill_stable_raw_order, int token_position, int n_swa); + +// Exact q=4 prefill must retain the q<=3 owner-kernel reduction order. The +// four-row dual-owner hybrid FFN path is numerically different from q=1 on the +// production target. +// Return the largest owner sub-batch allowed for this production intent. +int deepseek4_exact_prefill_hybrid_ffn_sub_batch( + bool exact_prefill_q1_ffn_order, + int n_tokens); bool deepseek4_snapshot_save(const DeepSeek4Cache & cache, ggml_backend_t snapshot_backend, DeepSeek4Snapshot & out); @@ -497,7 +505,8 @@ bool deepseek4_step_layer_range( MoeExpertComputeRuntime * expert_runtime = nullptr, MoeHybridRoutingStats * routing_stats = nullptr, bool execute_output_path = false, - bool exact_prefill_stable_raw_order = false); + bool exact_prefill_stable_raw_order = false, + bool exact_prefill_q1_ffn_order = false); bool build_deepseek4_moe_hybrid_storage_from_file( const std::string & path, diff --git a/server/tests/test_deepseek4_unit.cpp b/server/tests/test_deepseek4_unit.cpp index 63f5dd461..3b71d3feb 100644 --- a/server/tests/test_deepseek4_unit.cpp +++ b/server/tests/test_deepseek4_unit.cpp @@ -1136,6 +1136,18 @@ static void test_raw_ring_spans_after_wrap() { std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } +static void test_exact_prefill_hybrid_ffn_sub_batch() { + std::fprintf(stderr, " test_exact_prefill_hybrid_ffn_sub_batch ..."); + // Exact q4 keeps prompt geometry but evaluates independent hybrid-FFN + // owner rows as q3+q1. Every non-exact caller retains its prior q4 batch. + TEST_ASSERT(deepseek4_exact_prefill_hybrid_ffn_sub_batch(true, 4) == 3); + TEST_ASSERT(deepseek4_exact_prefill_hybrid_ffn_sub_batch(true, 3) == 3); + TEST_ASSERT(deepseek4_exact_prefill_hybrid_ffn_sub_batch(true, 1) == 1); + TEST_ASSERT(deepseek4_exact_prefill_hybrid_ffn_sub_batch(false, 4) == 4); + TEST_ASSERT(deepseek4_exact_prefill_hybrid_ffn_sub_batch(false, 5) == 5); + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); +} + struct ScopedEnvVar { explicit ScopedEnvVar(const char * name) : name(name ? name : ""), @@ -3969,6 +3981,7 @@ int main() { test_indexer_mask_cpu(backend); test_hash_routing_lookup(); test_raw_ring_spans_after_wrap(); + test_exact_prefill_hybrid_ffn_sub_batch(); test_auto_split_computation(); test_layer_range_validation(); test_hc_state_dimensions(); From 423bd172bcd91d5ac4092cd6c946375e05d5135e Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Tue, 11 Aug 2026 20:39:37 +0530 Subject: [PATCH 09/15] fix(dflash): retire exact route graph metadata --- server/src/deepseek4/deepseek4_graph.cpp | 20 ++++++++++++++++++-- server/src/deepseek4/deepseek4_internal.h | 2 ++ server/tests/test_deepseek4_unit.cpp | 2 ++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index c90ed4467..94a728861 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -213,6 +213,11 @@ int deepseek4_exact_prefill_hybrid_ffn_sub_batch( return exact_prefill_q1_ffn_order && n_tokens == 4 ? 3 : n_tokens; } +bool deepseek4_exact_prefill_route_graph_requires_eager( + bool exact_prefill_q1_ffn_order) { + return exact_prefill_q1_ffn_order; +} + struct DeepSeek4I32InputBinding { ggml_tensor * tensor = nullptr; int32_t value = 0; @@ -5565,8 +5570,19 @@ static bool eval_ds4_layer_range_hybrid_ffn( sizeof(float) * (size_t)n_embd * (size_t)n_tokens); } const auto route_compute_t0 = Ds4TimingClock::now(); - const bool route_ok = - ggml_backend_graph_compute(backend, gf) == GGML_STATUS_SUCCESS; + bool route_ok = false; + { + // This graph is rebuilt in temporary metadata for every exact-band + // step. Do not leave a native executable keyed to metadata that is + // immediately freed and may be recycled by a persistent owner graph. + // The scope ends before owner evaluation, so its stable graphs retain + // their normal replay behavior. + ScopedCudaGraphOverrides route_graph_scope( + deepseek4_exact_prefill_route_graph_requires_eager( + exact_prefill_q1_ffn_order)); + route_ok = + ggml_backend_graph_compute(backend, gf) == GGML_STATUS_SUCCESS; + } if (trace_prefill) { std::fprintf(stderr, "[deepseek4-prefill-trace] layer=%d ffn route compute=%s\n", diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index 5ef8f2014..143e4d463 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -351,6 +351,8 @@ bool deepseek4_exact_tokenwise_uses_runtime_raw_row( int deepseek4_exact_prefill_hybrid_ffn_sub_batch( bool exact_prefill_q1_ffn_order, int n_tokens); +bool deepseek4_exact_prefill_route_graph_requires_eager( + bool exact_prefill_q1_ffn_order); bool deepseek4_snapshot_save(const DeepSeek4Cache & cache, ggml_backend_t snapshot_backend, DeepSeek4Snapshot & out); diff --git a/server/tests/test_deepseek4_unit.cpp b/server/tests/test_deepseek4_unit.cpp index 3b71d3feb..056837e1c 100644 --- a/server/tests/test_deepseek4_unit.cpp +++ b/server/tests/test_deepseek4_unit.cpp @@ -1145,6 +1145,8 @@ static void test_exact_prefill_hybrid_ffn_sub_batch() { TEST_ASSERT(deepseek4_exact_prefill_hybrid_ffn_sub_batch(true, 1) == 1); TEST_ASSERT(deepseek4_exact_prefill_hybrid_ffn_sub_batch(false, 4) == 4); TEST_ASSERT(deepseek4_exact_prefill_hybrid_ffn_sub_batch(false, 5) == 5); + TEST_ASSERT(deepseek4_exact_prefill_route_graph_requires_eager(true)); + TEST_ASSERT(!deepseek4_exact_prefill_route_graph_requires_eager(false)); std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } From 7fe85b8f3a74db6509aa53ab15d07f13e758914f Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Sat, 15 Aug 2026 03:24:54 +0530 Subject: [PATCH 10/15] test(dflash): read balanced quota from correct slot --- server/test/test_moe_hybrid_storage.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/test/test_moe_hybrid_storage.cpp b/server/test/test_moe_hybrid_storage.cpp index b57ceb15a..be1ac56fb 100644 --- a/server/test/test_moe_hybrid_storage.cpp +++ b/server/test/test_moe_hybrid_storage.cpp @@ -70,7 +70,7 @@ TEST_CASE(MoeHybridStorageFixture, fractional_route_quota_rounds_over_the_batch) REQUIRE(owner_ids != nullptr); int32_t main_quota = 0; std::memcpy(&main_quota, - owner_ids->op_params + sizeof(int32_t), + owner_ids->op_params + 1, sizeof(main_quota)); REQUIRE(main_quota == 23); From ce291c633964b00adacc7d37cde266d0be5013c7 Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Mon, 17 Aug 2026 22:49:52 +0530 Subject: [PATCH 11/15] fix(hip): map no-device error in CUDA shim --- server/hip_compat/cuda_runtime.h | 1 + 1 file changed, 1 insertion(+) diff --git a/server/hip_compat/cuda_runtime.h b/server/hip_compat/cuda_runtime.h index b534d179c..56b66fec2 100644 --- a/server/hip_compat/cuda_runtime.h +++ b/server/hip_compat/cuda_runtime.h @@ -29,6 +29,7 @@ using cudaDeviceProp = hipDeviceProp_t; // Error codes #define cudaSuccess hipSuccess #define cudaErrorInvalidValue hipErrorInvalidValue +#define cudaErrorNoDevice hipErrorNoDevice #define cudaErrorIllegalAddress hipErrorIllegalAddress #define cudaErrorAssert hipErrorAssert #define cudaErrorLaunchFailure hipErrorLaunchFailure From dda094fb129270de8e263c7b9169f74d82207a6a Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Tue, 18 Aug 2026 05:25:45 +0530 Subject: [PATCH 12/15] fix(deepseek4): avoid cached graphs during snapshot prefill --- server/src/deepseek4/deepseek4_backend.cpp | 11 ++++++++++- server/src/deepseek4/deepseek4_backend.h | 2 ++ server/tests/test_deepseek4_unit.cpp | 3 +++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index 0872d534e..cf83425c7 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -1780,6 +1780,10 @@ int deepseek4_hybrid_prefill_chunk_tokens( : bounded; } +bool deepseek4_prefill_allows_decode_graph_reuse(bool save_snapshot) { + return !save_snapshot; +} + int DeepSeek4Backend::do_prefill(const std::vector & tokens, const DaemonIO & io, int kv_offset, @@ -1976,7 +1980,12 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, 0, w_.n_layer, logits_out, tokens.data() + i, timing ? &step_tel : nullptr, - /*allow_decode_graph_reuse=*/true, hp, + // Snapshot prefill owns transient checkpoint graph metadata. + // Keep it out of the persistent q=1 decode-graph cache so an + // executable cannot outlive its tensor-parent metadata. + /*allow_decode_graph_reuse=*/ + deepseek4_prefill_allows_decode_graph_reuse(save_snapshot), + hp, moe_hybrid_.get(), expert_runtime_.compute ? &expert_runtime_ : nullptr, routing_stats_.get(), output_intent.execute_output_path, diff --git a/server/src/deepseek4/deepseek4_backend.h b/server/src/deepseek4/deepseek4_backend.h index a9c58a18a..7729660b5 100644 --- a/server/src/deepseek4/deepseek4_backend.h +++ b/server/src/deepseek4/deepseek4_backend.h @@ -33,6 +33,8 @@ int deepseek4_hybrid_prefill_chunk_tokens( int context_end, int current_cap = 0); +bool deepseek4_prefill_allows_decode_graph_reuse(bool save_snapshot); + class DeepSeek4Backend : public ModelBackend { public: explicit DeepSeek4Backend(const DeepSeek4BackendConfig & cfg); diff --git a/server/tests/test_deepseek4_unit.cpp b/server/tests/test_deepseek4_unit.cpp index ade319225..47bc0a740 100644 --- a/server/tests/test_deepseek4_unit.cpp +++ b/server/tests/test_deepseek4_unit.cpp @@ -1764,6 +1764,9 @@ static void test_exact_prefill_chunk_policy() { PrefillAttentionMode::Sparse, false, /*batch_supported=*/true, 1024, 512) == 512); + TEST_ASSERT(deepseek4_prefill_allows_decode_graph_reuse(false)); + TEST_ASSERT(!deepseek4_prefill_allows_decode_graph_reuse(true)); + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } From 61909d5174d8071422a9c4c27f65b0b041329c27 Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Tue, 18 Aug 2026 13:59:13 +0530 Subject: [PATCH 13/15] fix(deepseek4): force eager snapshot prefill graphs --- server/src/deepseek4/deepseek4_backend.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index cf83425c7..bb71eaa77 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -5,6 +5,7 @@ #include "deepseek4_budget_hook.h" #include "deepseek4_internal.h" #include "common/dynamic_backend.h" +#include "common/cuda_graph_overrides.h" #include "common/peer_access.h" #include "common/sampler.h" @@ -1847,6 +1848,10 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, const bool save_snapshot = snap_slot >= 0 && snap_slot < PREFIX_SLOTS && snap_pos > kv_offset && snap_pos <= kv_offset + n_total; + // Snapshot construction owns transient checkpoint tensor metadata across + // thousands of prefill steps. Keep native HIP graph capture/replay eager + // for this scope so backend executables cannot retain those parent links. + ScopedCudaGraphOverrides snapshot_graph_scope(save_snapshot); // New sequence: clear the cache buffer so compressor state double-buffers // and compressed-KV rows start from zeros, exactly like a fresh server. // Without this, the first flush windows of a request pool over the From 5f7799bb034338e562761aa835cfdae3ccb09a73 Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Tue, 18 Aug 2026 16:36:39 +0530 Subject: [PATCH 14/15] fix(deepseek4): defer terminal snapshot save --- server/src/deepseek4/deepseek4_backend.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index bb71eaa77..b282129c6 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -1845,9 +1845,17 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, exact_bands_enabled && batch_supported && requested_chunk > 1 && chunk > 1; int pos = kv_offset; - const bool save_snapshot = + const bool snapshot_requested = snap_slot >= 0 && snap_slot < PREFIX_SLOTS && snap_pos > kv_offset && snap_pos <= kv_offset + n_total; + // A checkpoint at the terminal prompt boundary needs no special prefill + // graph or chunk boundary: the ordinary final chunk already commits the + // exact cache and logits that snapshot_save() records. Defer only that + // terminal save until the loop completes, while retaining the existing + // in-loop handling for checkpoints inside the prompt. + const bool terminal_snapshot = + snapshot_requested && snap_pos == kv_offset + n_total; + const bool save_snapshot = snapshot_requested && !terminal_snapshot; // Snapshot construction owns transient checkpoint tensor metadata across // thousands of prefill steps. Keep native HIP graph capture/replay eager // for this scope so backend executables cannot retain those parent links. @@ -2067,6 +2075,15 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, } } } + if (terminal_snapshot && pos == snap_pos) { + snapshot_saved = snapshot_save(snap_slot); + if (!snapshot_saved) { + std::fprintf(stderr, + "[deepseek4] failed to save terminal snapshot " + "slot=%d pos=%d\n", + snap_slot, snap_pos); + } + } keep_spec_feature_tail(spec_feat_window_, (size_t) std::max(0, w_.n_swa)); if (timing) { From 42d5c3e7f6672a840e2e1c4cbaf036a653f12804 Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Tue, 18 Aug 2026 21:21:35 +0530 Subject: [PATCH 15/15] fix(ds4): suppress inactive warning without verifier hooks --- server/src/deepseek4/deepseek4_graph.cpp | 4 ++-- server/tests/test_deepseek4_unit.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index d94952efa..292296fec 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -6893,8 +6893,8 @@ bool deepseek4_should_warn_fused_verify_inactive( bool fused_verify_enabled, bool fused_verify_candidate) { return fused_verify_enabled && !fused_verify_candidate && - n_tokens >= 2 && full_layer_range && - (!verify_hooks || verify_hooks->allow_fused_verify); + n_tokens >= 2 && full_layer_range && verify_hooks && + verify_hooks->allow_fused_verify; } DeepSeek4RecursiveOutputIntent deepseek4_recursive_output_intent( diff --git a/server/tests/test_deepseek4_unit.cpp b/server/tests/test_deepseek4_unit.cpp index f024fd092..1a6b86dbe 100644 --- a/server/tests/test_deepseek4_unit.cpp +++ b/server/tests/test_deepseek4_unit.cpp @@ -2027,7 +2027,7 @@ static void test_prefill_readout_lifecycle_and_fused_exclusion() { /*full_layer_range=*/true, /*fused_verify_enabled=*/true, /*fused_verify_candidate=*/false)); - TEST_ASSERT(deepseek4_should_warn_fused_verify_inactive( + TEST_ASSERT(!deepseek4_should_warn_fused_verify_inactive( /*n_tokens=*/4, /*verify_hooks=*/nullptr, /*full_layer_range=*/true, /*fused_verify_enabled=*/true,