diff --git a/server/docs/DS4.md b/server/docs/DS4.md index 3de39b571..e83a1d597 100644 --- a/server/docs/DS4.md +++ b/server/docs/DS4.md @@ -251,6 +251,7 @@ The runtime logs the chosen split with a `[deepseek4-split] auto-split:` banner. |----------|---------| | `DFLASH_DS4_CUDA_LAYERS` | Override the auto-split heuristic and pin the first `N` DeepSeek4 layers to CUDA. The remaining `43 - N` layers run on the Halo shard. | | `DFLASH_DS4_TIMING` | Enable DS4 timing logs for the layer-split parent and target-shard daemon. Useful for profiling prefill/decode breakdowns; leave unset for normal runs. | +| `DFLASH_DS4_EXACT_PREFILL_BANDS` | Opt in to compressor-safe exact prefill bands up to four tokens on supported layer-range paths. Exact attention remains tokenwise. Leave unset for the default single-token path; `--chunk 1` is the hard fallback. | | `DFLASH_DS4_ROCTX` | HIP-only, default-off semantic ROCTX ranges for an external rocprof trace. The library is loaded dynamically only when set to `1`, `true`, `yes`, or `on`. | | `DFLASH_DS4_SPEC` / `DFLASH_DS4_DRAFT` | Enable DSpark and select its GGUF. | | `DFLASH_DS4_DRAFT_BACKEND` / `DFLASH_DS4_DRAFT_GPU` | Backend and device for the in-process drafter. | diff --git a/server/docs/ENVIRONMENT.md b/server/docs/ENVIRONMENT.md index da7d8bb09..1e39b1167 100644 --- a/server/docs/ENVIRONMENT.md +++ b/server/docs/ENVIRONMENT.md @@ -100,6 +100,7 @@ consolidation of this list into CLI flags is tracked as follow-up work. - `DFLASH_DS4_DRAFT_BACKEND` - deepseek4_backend.cpp - `DFLASH_DS4_DRAFT_GPU` - deepseek4_backend.cpp - `DFLASH_DS4_DSPARK_DEBUG` - deepseek4_graph.cpp +- `DFLASH_DS4_EXACT_PREFILL_BANDS` - deepseek4_backend.cpp - `DFLASH_DS4_FUSED_VERIFY` - deepseek4_dspark_spec.cpp, deepseek4_loader.cpp - `DFLASH_DS4_HOTNESS_CSV` - deepseek4_backend.cpp - `DFLASH_DS4_MOE_TP` - deepseek4_backend.cpp diff --git a/server/hip_compat/cuda_runtime.h b/server/hip_compat/cuda_runtime.h index b534d179c..56b66fec2 100644 --- a/server/hip_compat/cuda_runtime.h +++ b/server/hip_compat/cuda_runtime.h @@ -29,6 +29,7 @@ using cudaDeviceProp = hipDeviceProp_t; // Error codes #define cudaSuccess hipSuccess #define cudaErrorInvalidValue hipErrorInvalidValue +#define cudaErrorNoDevice hipErrorNoDevice #define cudaErrorIllegalAddress hipErrorIllegalAddress #define cudaErrorAssert hipErrorAssert #define cudaErrorLaunchFailure hipErrorLaunchFailure diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index 7183c4876..5352e47c5 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -5,6 +5,7 @@ #include "deepseek4_budget_hook.h" #include "deepseek4_internal.h" #include "common/dynamic_backend.h" +#include "common/cuda_graph_overrides.h" #include "common/peer_access.h" #include "common/sampler.h" @@ -29,6 +30,10 @@ namespace dflash::common { +bool deepseek4_env_flag_value_enabled(const char * value) { + return value && value[0] && std::strcmp(value, "0") != 0; +} + namespace { using Clock = std::chrono::steady_clock; @@ -41,8 +46,7 @@ static uint64_t elapsed_us(Clock::time_point start, Clock::time_point end) { } static bool env_flag_enabled(const char * name) { - const char * value = std::getenv(name); - return value && value[0] && std::strcmp(value, "0") != 0; + return deepseek4_env_flag_value_enabled(std::getenv(name)); } static bool positive_env_double(const char * name, double fallback, @@ -666,6 +670,108 @@ static MoeLayerDesc make_ds4_expert_layer_desc(const DeepSeek4Layer & layer) { } // namespace +int deepseek4_prefill_chunk_tokens(PrefillAttentionMode mode, + bool exact_bands_enabled, + bool batch_supported, + int requested_chunk, + int layer_major_cap) { + if (!batch_supported || requested_chunk <= 1 || layer_major_cap <= 1) { + return 1; + } + + const int bounded_chunk = std::max( + 1, std::min(requested_chunk, layer_major_cap)); + if (mode == PrefillAttentionMode::Exact) { + constexpr int kMaxExactBandTokens = 4; + return exact_bands_enabled + ? std::min(bounded_chunk, kMaxExactBandTokens) + : 1; + } + return prefill_attention_mode_is_approximate(mode) ? bounded_chunk : 1; +} + +DeepSeek4PrefillOutputIntent deepseek4_prefill_output_intent( + PrefillAttentionMode mode, + bool exact_bands_active, + int n_tokens, + bool is_final_chunk, + bool ends_at_snapshot, + bool external_requires_logits) { + const bool legacy_readback = + !exact_bands_active && + (n_tokens == 1 || mode != PrefillAttentionMode::Exact); + const bool readback_logits = + is_final_chunk || ends_at_snapshot || external_requires_logits || + legacy_readback; + return { + /*execute_output_path=*/ + readback_logits || n_tokens == 1 || + mode != PrefillAttentionMode::Exact, + readback_logits, + }; +} + +void deepseek4_invalidate_prefill_logits_if_skipped( + bool readback_logits, + std::vector & last_logits, + int & last_logits_pos) { + if (readback_logits) return; + last_logits.clear(); + last_logits_pos = -1; +} + +DeepSeek4PrefillOutputIntent deepseek4_prepare_prefill_output_intent( + PrefillAttentionMode mode, + bool exact_bands_active, + int n_tokens, + bool is_final_chunk, + bool ends_at_snapshot, + bool external_requires_logits, + std::vector & last_logits, + int & last_logits_pos) { + const DeepSeek4PrefillOutputIntent intent = + deepseek4_prefill_output_intent( + mode, exact_bands_active, n_tokens, is_final_chunk, + ends_at_snapshot, external_requires_logits); + // This preparation function is the production ordering boundary: stale + // values are invalidated before the caller can enter any forward path. + deepseek4_invalidate_prefill_logits_if_skipped( + intent.readback_logits, last_logits, last_logits_pos); + return intent; +} + +bool deepseek4_commit_prefill_logits( + bool readback_logits, + int vocab_size, + int cache_position, + std::vector && logits, + std::vector & last_logits, + int & last_logits_pos) { + if (!readback_logits) return true; + if (vocab_size <= 0 || logits.size() != (size_t) vocab_size) { + last_logits.clear(); + last_logits_pos = -1; + return false; + } + last_logits = std::move(logits); + last_logits_pos = cache_position; + return true; +} + +Ds4VerifyHooks deepseek4_make_prefill_capture_hooks( + const std::vector * capture_layer_ids, + std::vector * capture_out, + int capture_token_begin, + int capture_token_end) { + Ds4VerifyHooks hooks; + hooks.capture_layer_ids = capture_layer_ids; + hooks.capture_out = capture_out; + hooks.capture_token_begin = capture_token_begin; + hooks.capture_token_end = capture_token_end; + hooks.allow_fused_verify = false; + return hooks; +} + DeepSeek4Backend::DeepSeek4Backend(const DeepSeek4BackendConfig & cfg) : cfg_(cfg) {} @@ -1675,6 +1781,10 @@ int deepseek4_hybrid_prefill_chunk_tokens( : bounded; } +bool deepseek4_prefill_allows_decode_graph_reuse(bool save_snapshot) { + return !save_snapshot; +} + int DeepSeek4Backend::do_prefill(const std::vector & tokens, const DaemonIO & io, int kv_offset, @@ -1691,25 +1801,28 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, // that snapshot plus the current ubatch, and commits only the final SWA // tail. Learned compressor boundaries are emitted inside the same graph. // - // Mixed hot/cold hybrid execution still has single-token HC semantics, so - // retain the reference path there. --chunk 1 is the explicit fallback. + // Mixed hot/cold hybrid execution without the layer-range runtime still + // has single-token HC semantics. --chunk 1 is the explicit fallback for + // every path. const int requested_chunk = cfg_.chunk > 0 ? cfg_.chunk : w_.n_swa; const int n_total = (int)tokens.size(); // Bound the layer-major graph to the topology validated by the prefill // kernels. Smaller tail chunks use the same scheduler or its reference // fallback. const int layer_major_cap = DS4_MAX_LAYER_MAJOR_PREFILL_TOKENS; - // Only sparse prefill has a qualified batched mixed-owner HC path. Dense - // hybrid execution remains tokenwise; batching it would skip per-token HC - // post-mixing and corrupt the hidden state. - const bool hybrid_batch_supported = - !moe_hybrid_ || cfg_.prefill_mode == PrefillAttentionMode::Sparse; - const int base_chunk = - !prefill_attention_mode_is_approximate(cfg_.prefill_mode) || - !hybrid_batch_supported - ? 1 - : std::max(1, std::min(requested_chunk, - layer_major_cap)); + const bool layer_range_hybrid = + moe_hybrid_ && (expert_runtime_.compute || expert_backend_); + const bool batch_supported = + !moe_hybrid_ || + cfg_.prefill_mode == PrefillAttentionMode::Sparse || + (cfg_.prefill_mode == PrefillAttentionMode::Exact && + layer_range_hybrid); + const bool exact_bands_enabled = + env_flag_enabled("DFLASH_DS4_EXACT_PREFILL_BANDS"); + const int base_chunk = deepseek4_prefill_chunk_tokens( + cfg_.prefill_mode, + exact_bands_enabled, + batch_supported, requested_chunk, layer_major_cap); const bool bound_hybrid_scratch = moe_hybrid_ && cfg_.prefill_mode == PrefillAttentionMode::Sparse; @@ -1722,17 +1835,31 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, hybrid_prefill_chunk_cap_ = hybrid_prefill_chunk_cap_ > 0 ? std::min(hybrid_prefill_chunk_cap_, chunk) : chunk; - } - if (chunk < base_chunk) { std::fprintf(stderr, "[deepseek4] hybrid prefill scratch bound: " "chunk %d->%d for context_end=%d (sticky)\n", base_chunk, chunk, kv_offset + n_total); } + const bool exact_bands_active = + cfg_.prefill_mode == PrefillAttentionMode::Exact && + exact_bands_enabled && batch_supported && requested_chunk > 1 && + chunk > 1; int pos = kv_offset; - const bool save_snapshot = + const bool snapshot_requested = snap_slot >= 0 && snap_slot < PREFIX_SLOTS && snap_pos > kv_offset && snap_pos <= kv_offset + n_total; + // A checkpoint at the terminal prompt boundary needs no special prefill + // graph or chunk boundary: the ordinary final chunk already commits the + // exact cache and logits that snapshot_save() records. Defer only that + // terminal save until the loop completes, while retaining the existing + // in-loop handling for checkpoints inside the prompt. + const bool terminal_snapshot = + snapshot_requested && snap_pos == kv_offset + n_total; + const bool save_snapshot = snapshot_requested && !terminal_snapshot; + // Snapshot construction owns transient checkpoint tensor metadata across + // thousands of prefill steps. Keep native HIP graph capture/replay eager + // for this scope so backend executables cannot retain those parent links. + ScopedCudaGraphOverrides snapshot_graph_scope(save_snapshot); // New sequence: clear the cache buffer so compressor state double-buffers // and compressed-KV rows start from zeros, exactly like a fresh server. // Without this, the first flush windows of a request pool over the @@ -1813,20 +1940,34 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, DeepSeek4StepTelemetry step_tel; if (timing) step_tel.embed_us = elapsed_us(embed_t0, Clock::now()); + const bool capture_final = i + n_tok > spec_final_from; + const bool capture_snapshot = + !snapshot_saved && i < spec_snap_to && + i + n_tok > spec_snap_from; + const bool capture_requested = + spec_enabled_ && spec_drafter_ && + (capture_final || capture_snapshot); + const bool ends_at_snapshot = + save_snapshot && !snapshot_saved && pos + n_tok == snap_pos; + // Execution topology and host vocabulary readback are independent. + // An exact-band singleton leaf keeps the established q=1/fused graph, + // but only a final/snapshot/external consumer receives host logits. + // Capture hooks request feature rows, not vocabulary values. + const DeepSeek4PrefillOutputIntent output_intent = + deepseek4_prepare_prefill_output_intent( + cfg_.prefill_mode, exact_bands_active, n_tok, + i + n_tok == n_total, ends_at_snapshot, + /*external_requires_logits=*/false, + last_logits_, last_logits_pos_); std::vector logits; + std::vector * logits_out = + output_intent.readback_logits ? &logits : nullptr; bool ok = false; std::vector hc_state; Ds4VerifyHooks spec_hooks; std::vector spec_cap; Ds4VerifyHooks * hp = nullptr; - const bool capture_final = i + n_tok > spec_final_from; - const bool capture_snapshot = - !snapshot_saved && i < spec_snap_to && - i + n_tok > spec_snap_from; - if (spec_enabled_ && spec_drafter_ && - (capture_final || capture_snapshot)) { - spec_hooks.capture_layer_ids = &spec_drafter_->capture_layer_ids; - spec_hooks.capture_out = &spec_cap; + if (capture_requested) { int capture_begin = n_tok; int capture_end = 0; if (capture_final) { @@ -1840,21 +1981,28 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, capture_end = std::max( capture_end, std::min(n_tok, spec_snap_to - i)); } - spec_hooks.capture_token_begin = capture_begin; - spec_hooks.capture_token_end = capture_end; + spec_hooks = deepseek4_make_prefill_capture_hooks( + &spec_drafter_->capture_layer_ids, &spec_cap, + capture_begin, capture_end); hp = &spec_hooks; } if (moe_hybrid_ && (expert_runtime_.compute || expert_backend_)) { ok = deepseek4_step_layer_range( backend_, cfg_.device.gpu, w_, cache_, hc_state, embed.data(), n_tok, pos, - 0, w_.n_layer, &logits, + 0, w_.n_layer, logits_out, tokens.data() + i, timing ? &step_tel : nullptr, - /*allow_decode_graph_reuse=*/true, hp, + // Snapshot prefill owns transient checkpoint graph metadata. + // Keep it out of the persistent q=1 decode-graph cache so an + // executable cannot outlive its tensor-parent metadata. + /*allow_decode_graph_reuse=*/ + deepseek4_prefill_allows_decode_graph_reuse(save_snapshot), + hp, moe_hybrid_.get(), expert_runtime_.compute ? &expert_runtime_ : nullptr, - routing_stats_.get()); + routing_stats_.get(), output_intent.execute_output_path, + exact_bands_active, exact_bands_active); } else if (moe_hybrid_) { ok = deepseek4_step(backend_, cfg_.device.gpu, w_, cache_, embed.data(), n_tok, pos, logits, moe_hybrid_.get(), tokens.data() + i, @@ -1864,12 +2012,15 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, hp, expert_runtime_.compute ? &expert_runtime_ : nullptr); } else { - ok = deepseek4_step_layer_range(backend_, cfg_.device.gpu, w_, cache_, hc_state, - embed.data(), n_tok, pos, - 0, w_.n_layer, &logits, - tokens.data() + i, - timing ? &step_tel : nullptr, - cfg_.prefill_mode != PrefillAttentionMode::Sparse, hp); + ok = deepseek4_step_layer_range( + backend_, cfg_.device.gpu, w_, cache_, hc_state, + embed.data(), n_tok, pos, 0, w_.n_layer, logits_out, + tokens.data() + i, timing ? &step_tel : nullptr, + cfg_.prefill_mode != PrefillAttentionMode::Sparse, hp, + /*moe_hybrid=*/nullptr, /*expert_runtime=*/nullptr, + /*routing_stats=*/nullptr, + output_intent.execute_output_path, + exact_bands_active, exact_bands_active); } if (ok && hp && !spec_cap.empty()) { const int feat_row = spec_drafter_->n_target_layers * w_.n_embd; @@ -1893,9 +2044,15 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, add_step_tel(tel_acc, step_tel); steps++; } - last_logits_ = std::move(logits); pos += n_tok; - last_logits_pos_ = cache_.cur_pos; + if (!deepseek4_commit_prefill_logits( + output_intent.readback_logits, w_.n_vocab, cache_.cur_pos, + std::move(logits), last_logits_, last_logits_pos_)) { + std::fprintf(stderr, + "[deepseek4] invalid prefill logits at pos=%d\n", + cache_.cur_pos); + return -1; + } i += n_tok; if (save_snapshot && !snapshot_saved && pos == snap_pos) { snapshot_saved = snapshot_save(snap_slot); @@ -1918,6 +2075,15 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, } } } + if (terminal_snapshot && pos == snap_pos) { + snapshot_saved = snapshot_save(snap_slot); + if (!snapshot_saved) { + std::fprintf(stderr, + "[deepseek4] failed to save terminal snapshot " + "slot=%d pos=%d\n", + snap_slot, snap_pos); + } + } keep_spec_feature_tail(spec_feat_window_, (size_t) std::max(0, w_.n_swa)); if (timing) { @@ -1963,7 +2129,17 @@ bool DeepSeek4Backend::do_decode(int committed, int n_gen, // Get last logits and sample std::vector logits; - if (generated == 0 && !last_logits_.empty()) { + if (generated == 0 && committed > 0) { + if (last_logits_.size() != (size_t) w_.n_vocab || + last_logits_pos_ != committed || + last_logits_pos_ != cache_.cur_pos) { + std::fprintf(stderr, + "[deepseek4] refusing missing or stale prefill logits " + "(logits_pos=%d committed=%d cache_pos=%d size=%zu)\n", + last_logits_pos_, committed, cache_.cur_pos, + last_logits_.size()); + return false; + } logits = last_logits_; } else { std::vector embed(w_.n_embd); @@ -2147,8 +2323,11 @@ GenerateResult DeepSeek4Backend::generate_from_state( } if (spec_enabled_ && spec_drafter_ && req.n_gen > 0 && !req.force_ar_decode && !budget_requires_ar && !sampling_requires_ar) { - if (last_logits_.empty()) { - result.fail(GenerateErrorCode::DecodeFailed, "spec: no prefill logits"); + if (last_logits_.size() != (size_t) w_.n_vocab || + last_logits_pos_ != committed || + last_logits_pos_ != cache_.cur_pos) { + result.fail(GenerateErrorCode::DecodeFailed, + "spec: missing or stale prefill logits"); return result; } int seed = 0; diff --git a/server/src/deepseek4/deepseek4_backend.h b/server/src/deepseek4/deepseek4_backend.h index a9c58a18a..7729660b5 100644 --- a/server/src/deepseek4/deepseek4_backend.h +++ b/server/src/deepseek4/deepseek4_backend.h @@ -33,6 +33,8 @@ int deepseek4_hybrid_prefill_chunk_tokens( int context_end, int current_cap = 0); +bool deepseek4_prefill_allows_decode_graph_reuse(bool save_snapshot); + class DeepSeek4Backend : public ModelBackend { public: explicit DeepSeek4Backend(const DeepSeek4BackendConfig & cfg); diff --git a/server/src/deepseek4/deepseek4_fused_verify.inc b/server/src/deepseek4/deepseek4_fused_verify.inc index f831f2b68..25450b782 100644 --- a/server/src/deepseek4/deepseek4_fused_verify.inc +++ b/server/src/deepseek4/deepseek4_fused_verify.inc @@ -1024,7 +1024,7 @@ static int ds4_try_fused_verify_step( const float * embed, int n_tokens, int kv_start, - std::vector & out_logits, + std::vector * out_logits, const int32_t * token_ids, Ds4VerifyHooks * hooks, DeepSeek4StepTelemetry * telemetry, @@ -1342,13 +1342,13 @@ static int ds4_try_fused_verify_step( ggml_backend_tensor_get(fg->logits, hooks->all_logits_out->data(), 0, sizeof(float) * (size_t) w.n_vocab * q); } - if (!argmax_only) { - out_logits.resize((size_t) w.n_vocab); - ggml_backend_tensor_get(fg->logits, out_logits.data(), + if (!argmax_only && out_logits) { + out_logits->resize((size_t) w.n_vocab); + ggml_backend_tensor_get(fg->logits, out_logits->data(), (size_t) (q - 1) * (size_t) w.n_vocab * sizeof(float), sizeof(float) * (size_t) w.n_vocab); - } else { - out_logits.clear(); + } else if (out_logits) { + out_logits->clear(); } if (hooks->capture_out && ex->capture && ncap > 0) { hooks->capture_out->resize((size_t) ncap * w.n_embd * q); @@ -1356,7 +1356,9 @@ static int ds4_try_fused_verify_step( ex->capture, hooks->capture_out->data(), 0, sizeof(float) * hooks->capture_out->size()); } - if (telemetry) { + if (telemetry && + (argmax_only || hooks->all_logits_out || out_logits || + (hooks->capture_out && ex->capture && ncap > 0))) { telemetry->full_graph_read_us += ds4_elapsed_us(read_t0, Ds4TimingClock::now()); } return 1; diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index f9c0c00b5..292296fec 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -197,6 +197,28 @@ int deepseek4_previous_raw_ring_spans( return count; } +bool deepseek4_exact_tokenwise_uses_runtime_raw_row( + bool exact_prefill_stable_raw_order, + int token_position, + int n_swa) { + return exact_prefill_stable_raw_order && n_swa > 0 && + token_position >= n_swa - 1; +} + +int deepseek4_exact_prefill_hybrid_ffn_sub_batch( + bool exact_prefill_q1_ffn_order, + int n_tokens) { + // q=1..3 use the same reduced-stack MMVQ reduction order on gfx1151. + // Exactly four rows select a different kernel topology, so retain q=4 + // prompt geometry while evaluating its independent FFN rows as q=3+q=1. + return exact_prefill_q1_ffn_order && n_tokens == 4 ? 3 : n_tokens; +} + +bool deepseek4_exact_prefill_route_graph_requires_eager( + bool exact_prefill_q1_ffn_order) { + return exact_prefill_q1_ffn_order; +} + struct DeepSeek4I32InputBinding { ggml_tensor * tensor = nullptr; int32_t value = 0; @@ -3831,12 +3853,13 @@ static void hc_pre_auto_into(float * working, n_embd, n_hc, sinkhorn_iters, hc_eps, flat, mix_scratch, serial_fn); } -static void hc_pre_batch(std::vector & working, +static bool hc_pre_batch(std::vector & working, std::vector & post, std::vector & comb, const float * hc_state, const HcWeightsCpu & weights, ggml_tensor * fn_tensor, + int device, int n_tokens, int n_embd, int n_hc, @@ -3847,7 +3870,20 @@ static void hc_pre_batch(std::vector & working, post.resize((size_t)n_tokens * (size_t)n_hc); comb.resize((size_t)n_tokens * (size_t)n_hc * (size_t)n_hc); + std::atomic device_ready{true}; ds4_pool_for_tokens(n_tokens, [&](int t0, int t1) { +#if defined(DFLASH27B_BACKEND_CUDA) + thread_local int selected_device = -1; + if (selected_device != device) { + if (!deepseek4_cuda_hc_set_device(device)) { + device_ready.store(false, std::memory_order_relaxed); + return; + } + selected_device = device; + } +#else + (void) device; +#endif std::vector flat(hc_dim); float mix[24]; for (int t = t0; t < t1; ++t) { @@ -3866,6 +3902,7 @@ static void hc_pre_batch(std::vector & working, /*serial_fn=*/n_tokens > 1); } }); + return device_ready.load(std::memory_order_relaxed); } static void cpu_hc_post(float * out_hc, const float * block_out, @@ -5358,8 +5395,9 @@ static bool ds4_build_fused_decode_graph( #include "deepseek4_fused_verify.inc" -// Returns 1 on success (out_logits filled), 0 to fall back to the per-layer -// path, -1 on a hard failure after cache state may have been touched. +// Returns 1 on success, 0 to fall back to the per-layer path, or -1 on a hard +// failure after cache state may have been touched. The output graph always +// executes; a null out_logits suppresses only the host vocabulary readback. static int ds4_try_fused_decode_step( DeepSeek4FusedDecodeCache & fc, ggml_backend_t backend, @@ -5371,7 +5409,7 @@ static int ds4_try_fused_decode_step( std::vector & hash_scratch, const float * embed, int kv_start, - std::vector & out_logits, + std::vector * out_logits, const int32_t * token_ids, DeepSeek4StepTelemetry * telemetry) { if (fc.disabled) return 0; @@ -5548,11 +5586,16 @@ static int ds4_try_fused_decode_step( if (telemetry) telemetry->full_graph_compute_us += ds4_elapsed_us(compute_t0, Ds4TimingClock::now()); // ── Read logits ───────────────────────────────────────────────── - const auto read_t0 = Ds4TimingClock::now(); - out_logits.resize((size_t) w.n_vocab); - ggml_backend_tensor_get(fg->logits, out_logits.data(), 0, - sizeof(float) * (size_t) w.n_vocab); - if (telemetry) telemetry->full_graph_read_us += ds4_elapsed_us(read_t0, Ds4TimingClock::now()); + if (out_logits) { + const auto read_t0 = Ds4TimingClock::now(); + out_logits->resize((size_t) w.n_vocab); + ggml_backend_tensor_get(fg->logits, out_logits->data(), 0, + sizeof(float) * (size_t) w.n_vocab); + if (telemetry) { + telemetry->full_graph_read_us += + ds4_elapsed_us(read_t0, Ds4TimingClock::now()); + } + } return 1; } @@ -5571,6 +5614,7 @@ static bool eval_ds4_layer_range_hybrid_ffn( MoeHybridRoutingStats * routing_stats, std::vector & out, DeepSeek4StepTelemetry * telemetry, + bool exact_prefill_q1_ffn_order, const MoeHybridDeviceOutputs * device_outputs = nullptr) { const bool trace_prefill = ds4_env_flag("DFLASH_DS4_PREFILL_TRACE"); if (trace_prefill) { @@ -5689,8 +5733,19 @@ static bool eval_ds4_layer_range_hybrid_ffn( sizeof(float) * (size_t)n_embd * (size_t)n_tokens); } const auto route_compute_t0 = Ds4TimingClock::now(); - const bool route_ok = - ggml_backend_graph_compute(backend, gf) == GGML_STATUS_SUCCESS; + bool route_ok = false; + { + // This graph is rebuilt in temporary metadata for every exact-band + // step. Do not leave a native executable keyed to metadata that is + // immediately freed and may be recycled by a persistent owner graph. + // The scope ends before owner evaluation, so its stable graphs retain + // their normal replay behavior. + ScopedCudaGraphOverrides route_graph_scope( + deepseek4_exact_prefill_route_graph_requires_eager( + exact_prefill_q1_ffn_order)); + route_ok = + ggml_backend_graph_compute(backend, gf) == GGML_STATUS_SUCCESS; + } if (trace_prefill) { std::fprintf(stderr, "[deepseek4-prefill-trace] layer=%d ffn route compute=%s\n", @@ -5813,16 +5868,48 @@ static bool eval_ds4_layer_range_hybrid_ffn( layer); } const auto owners_t0 = Ds4TimingClock::now(); - const bool ok = eval_ds4_hybrid( - backend, hybrid.cpu_backend, cfg, desc, &hybrid, - hybrid.layers[(size_t)layer], nullptr, - layer, n_embd, route_width, - device_ffn_input ? nullptr : normed_host.data(), - selected.data(), weights.data(), - n_tokens, out, hot_alloc, cold_alloc, - expert_compute, expert_layer, telemetry, - device_ffn_input ? normed : nullptr, - device_ffn_input ? device_outputs : nullptr); + const int owner_sub_batch = + deepseek4_exact_prefill_hybrid_ffn_sub_batch( + exact_prefill_q1_ffn_order, n_tokens); + bool ok = true; + if (owner_sub_batch > 0 && owner_sub_batch < n_tokens) { + out.resize((size_t)n_embd * (size_t)n_tokens); + std::vector sub_out; + for (int token_begin = 0; token_begin < n_tokens; + token_begin += owner_sub_batch) { + const int token_count = + std::min(owner_sub_batch, n_tokens - token_begin); + ok = eval_ds4_hybrid( + backend, hybrid.cpu_backend, cfg, desc, &hybrid, + hybrid.layers[(size_t)layer], nullptr, + layer, n_embd, route_width, + normed_host.data() + (size_t)token_begin * (size_t)n_embd, + selected.data() + (size_t)token_begin * (size_t)route_width, + weights.data() + (size_t)token_begin * (size_t)route_width, + token_count, sub_out, /*hot_alloc=*/nullptr, + /*cold_alloc=*/nullptr, expert_compute, expert_layer, + telemetry); + if (!ok || sub_out.size() != + (size_t)n_embd * (size_t)token_count) { + ok = false; + break; + } + std::memcpy( + out.data() + (size_t)token_begin * (size_t)n_embd, + sub_out.data(), sizeof(float) * sub_out.size()); + } + } else { + ok = eval_ds4_hybrid( + backend, hybrid.cpu_backend, cfg, desc, &hybrid, + hybrid.layers[(size_t)layer], nullptr, + layer, n_embd, route_width, + device_ffn_input ? nullptr : normed_host.data(), + selected.data(), weights.data(), + n_tokens, out, hot_alloc, cold_alloc, + expert_compute, expert_layer, telemetry, + device_ffn_input ? normed : nullptr, + device_ffn_input ? device_outputs : nullptr); + } if (trace_prefill) { std::fprintf(stderr, "[deepseek4-prefill-trace] layer=%d expert owners=%s " @@ -5847,6 +5934,7 @@ static bool ds4_run_exact_tokenwise_prefill_attention( int n_tokens, int kv_start, DeepSeek4AttentionImpl attention_impl, + bool exact_prefill_stable_raw_order, std::vector & attn_out_host, DeepSeek4CachedLayerAlloc & attn_alloc, DeepSeek4StepTelemetry * telemetry) { @@ -5868,11 +5956,25 @@ static bool ds4_run_exact_tokenwise_prefill_attention( std::vector i32_array_inputs; std::vector i64_array_inputs; std::vector f32_array_inputs; + // q=1 switches to a stable physical-ring reduction once the SWA + // window fills. Exact q=2..4 must use the same runtime row topology; + // reconstructing chronological spans changes the F32 reduction order + // after wrap and can move learned routing weights past the oracle + // tolerance even though the raw/compressed cache contents are equal. + DeepSeek4AttentionGraphInputs stable_inputs{}; + const int token_position = kv_start + ti; + if (deepseek4_exact_tokenwise_uses_runtime_raw_row( + exact_prefill_stable_raw_order, token_position, w.n_swa)) { + stable_inputs.raw_kv_rows = + ggml_new_tensor_2d(ctx, GGML_TYPE_I64, 1, 1); + ggml_set_input(stable_inputs.raw_kv_rows); + } ggml_cgraph * gf = ggml_new_graph_custom( ctx, ds4_attn_step_graph_size(1), false); ggml_tensor * normed = build_rms_norm(ctx, inp, L.attn_norm, w.rms_eps); ggml_tensor * attn_out = build_mla_attention( - ctx, gf, normed, w, L, lc, il, kv_start + ti, 1, nullptr, + ctx, gf, normed, w, L, lc, il, token_position, 1, + stable_inputs.raw_kv_rows ? &stable_inputs : nullptr, i32_inputs, i32_array_inputs, i64_array_inputs, &f32_array_inputs, attention_impl); ggml_set_output(attn_out); @@ -5899,6 +6001,11 @@ static bool ds4_run_exact_tokenwise_prefill_attention( ggml_backend_tensor_set(inp, cur + (size_t) ti * n_embd, 0, sizeof(float) * (size_t) n_embd); + if (stable_inputs.raw_kv_rows) { + const int64_t raw_row = token_position % w.n_swa; + ggml_backend_tensor_set( + stable_inputs.raw_kv_rows, &raw_row, 0, sizeof(raw_row)); + } for (const auto & b : i32_inputs) { ggml_backend_tensor_set(b.tensor, &b.value, 0, sizeof(b.value)); } @@ -6748,6 +6855,77 @@ static bool initialize_layer_range_cache( runtime.owns_output = owns_output; return true; } + +bool deepseek4_should_attempt_fused_verify( + int n_tokens, + const Ds4VerifyHooks * verify_hooks, + bool owner_topology_supported, + bool full_layer_range, + bool execute_output_path, + bool gpu_backend, + bool fused_verify_enabled) { + return owner_topology_supported && n_tokens >= 2 && + n_tokens <= DS4_CONSERVATIVE_VERIFY_MAX_TOKENS && + verify_hooks && verify_hooks->allow_fused_verify && + full_layer_range && execute_output_path && gpu_backend && + fused_verify_enabled; +} + +bool deepseek4_should_attempt_wide_fused_verify( + int n_tokens, + const Ds4VerifyHooks * verify_hooks, + bool q5_enabled, + bool owner_topology_supported, + bool full_layer_range, + bool has_output_storage, + bool gpu_backend, + bool fused_verify_enabled) { + return n_tokens == DS4_Q5_VERIFY_TOKENS && q5_enabled && + verify_hooks && verify_hooks->allow_fused_verify && + owner_topology_supported && full_layer_range && + has_output_storage && gpu_backend && fused_verify_enabled; +} + +bool deepseek4_should_warn_fused_verify_inactive( + int n_tokens, + const Ds4VerifyHooks * verify_hooks, + bool full_layer_range, + bool fused_verify_enabled, + bool fused_verify_candidate) { + return fused_verify_enabled && !fused_verify_candidate && + n_tokens >= 2 && full_layer_range && verify_hooks && + verify_hooks->allow_fused_verify; +} + +DeepSeek4RecursiveOutputIntent deepseek4_recursive_output_intent( + PrefillAttentionMode mode, + bool parent_execute_output_path, + bool parent_has_output_storage, + bool is_last_shard, + int chunk_tokens, + bool is_final_chunk) { + const bool exact = mode == PrefillAttentionMode::Exact; + return { + /*execute_output_path=*/exact + ? chunk_tokens == 1 || + (is_final_chunk && parent_execute_output_path) + : parent_execute_output_path, + /*pass_output_storage=*/ + parent_has_output_storage && + (!is_last_shard || !exact || is_final_chunk), + }; +} + +bool deepseek4_should_attempt_fused_hybrid_decode( + bool fused_hybrid_decode, + bool full_layer_range, + bool execute_output_path, + bool gpu_backend, + bool fused_verify_enabled) { + return fused_hybrid_decode && full_layer_range && execute_output_path && + gpu_backend && fused_verify_enabled; +} + bool deepseek4_step_layer_range( ggml_backend_t backend, int device, @@ -6766,7 +6944,10 @@ bool deepseek4_step_layer_range( Ds4VerifyHooks * verify_hooks, MoeHybridStorage * moe_hybrid, MoeExpertComputeRuntime * expert_runtime, - MoeHybridRoutingStats * routing_stats) { + MoeHybridRoutingStats * routing_stats, + bool execute_output_path, + bool exact_prefill_stable_raw_order, + bool exact_prefill_q1_ffn_order) { const auto step_t0 = Ds4TimingClock::now(); if (!deepseek4_cuda_hc_set_device(device)) { @@ -6781,41 +6962,58 @@ bool deepseek4_step_layer_range( const int n_hc = w.n_hc; const int hc_dim = n_hc * n_embd; const bool is_last_shard = (layer_end >= w.n_layer); + const bool readback_logits = out_logits != nullptr; + const bool external_output_consumer = + verify_hooks && + (verify_hooks->all_logits_out || verify_hooks->argmax_out); + execute_output_path = + execute_output_path || readback_logits || external_output_consumer; const bool fused_hybrid_ready = moe_hybrid && !expert_runtime && moe_hybrid->materialized_cold_experts && moe_hybrid->cold_backend_kind == MoeHybridColdBackend::Gpu && moe_hybrid->cold_backend && moe_hybrid->cold_backend != backend; - const bool wide_verify_candidate = - n_tokens == DS4_Q5_VERIFY_TOKENS && - ds4_env_flag("DFLASH_DS4_Q5_VERIFY"); + const bool conservative_verify_candidate = + deepseek4_should_attempt_fused_verify( + n_tokens, verify_hooks, + !moe_hybrid || fused_hybrid_ready, + layer_begin == 0 && is_last_shard, + execute_output_path, + ds4_backend_is_gpu(backend), ds4_fused_verify_enabled()); + const bool q5_verify_enabled = ds4_env_flag("DFLASH_DS4_Q5_VERIFY"); const bool fused_verify_candidate = - (!moe_hybrid || fused_hybrid_ready) && - n_tokens >= 2 && - (n_tokens <= DS4_CONSERVATIVE_VERIFY_MAX_TOKENS || - wide_verify_candidate) && verify_hooks && - layer_begin == 0 && is_last_shard && out_logits && - ds4_backend_is_gpu(backend) && ds4_fused_verify_enabled(); + conservative_verify_candidate || + deepseek4_should_attempt_wide_fused_verify( + n_tokens, verify_hooks, + q5_verify_enabled, + !moe_hybrid || fused_hybrid_ready, + layer_begin == 0 && is_last_shard, out_logits != nullptr, + ds4_backend_is_gpu(backend), ds4_fused_verify_enabled()); // Fused verify has many preconditions and declining any of them is // invisible: the request still decodes, still reports a healthy acceptance // rate, and only the throughput differs. Name the failed condition once so // a slow DSpark run can be attributed from the log instead of guessed at. // (No run has yet tripped this on gfx1151 — it is here so that the next // "spec decode is slow" report starts from evidence.) - if (ds4_fused_verify_enabled() && !fused_verify_candidate && - n_tokens >= 2 && layer_begin == 0 && is_last_shard) { + if (deepseek4_should_warn_fused_verify_inactive( + n_tokens, verify_hooks, layer_begin == 0 && is_last_shard, + ds4_fused_verify_enabled(), fused_verify_candidate)) { static bool warned = false; if (!warned) { warned = true; std::fprintf(stderr, "[deepseek4] DFLASH_DS4_FUSED_VERIFY=1 but fused verify is " - "inactive: n_tokens=%d (cap %d) verify_hooks=%d out_logits=%d " - "backend_gpu=%d moe_hybrid=%d expert_runtime=%d " + "inactive: n_tokens=%d (cap %d) verify_hooks=%d " + "allow_fused_verify=%d execute_output_path=%d out_logits=%d " + "q5_enabled=%d backend_gpu=%d moe_hybrid=%d expert_runtime=%d " "materialized_cold=%d cold_backend_kind_gpu=%d " "cold_backend_distinct=%d; verify falls back to the dense " "full-expert path\n", n_tokens, GGML_CUDA_DS4_MIX_MMV_MAX_TOKENS, - verify_hooks ? 1 : 0, out_logits ? 1 : 0, + verify_hooks ? 1 : 0, + verify_hooks && verify_hooks->allow_fused_verify ? 1 : 0, + execute_output_path ? 1 : 0, out_logits ? 1 : 0, + q5_verify_enabled ? 1 : 0, ds4_backend_is_gpu(backend) ? 1 : 0, moe_hybrid ? 1 : 0, expert_runtime ? 1 : 0, moe_hybrid && moe_hybrid->materialized_cold_experts ? 1 : 0, @@ -6895,20 +7093,33 @@ bool deepseek4_step_layer_range( chunk_hooks.capture_layer_ids = verify_hooks->capture_layer_ids; chunk_hooks.capture_out = verify_hooks->capture_out ? &chunk_capture : nullptr; chunk_hooks.all_logits_out = verify_hooks->all_logits_out ? &chunk_logits : nullptr; + chunk_hooks.allow_fused_verify = verify_hooks->allow_fused_verify; chunk_hooks_ptr = &chunk_hooks; } + const bool final_chunk = off + chunk == n_tokens; + const DeepSeek4RecursiveOutputIntent chunk_intent = + deepseek4_recursive_output_intent( + cache.prefill_mode, execute_output_path, + out_logits != nullptr, is_last_shard, chunk, final_chunk); + std::vector * chunk_output = nullptr; + if (chunk_intent.pass_output_storage) { + chunk_output = &chunk_out; + } if (!deepseek4_step_layer_range( backend, device, w, cache, chunk_hc, embed + (size_t) off * input_width, chunk, kv_start + off, layer_begin, layer_end, - out_logits ? &chunk_out : nullptr, + chunk_output, token_ids ? token_ids + off : nullptr, telemetry, allow_decode_graph_reuse, chunk_hooks_ptr, - moe_hybrid, expert_runtime, routing_stats)) { + moe_hybrid, expert_runtime, routing_stats, + chunk_intent.execute_output_path, + exact_prefill_stable_raw_order, + exact_prefill_q1_ffn_order)) { return false; } hc_all.insert(hc_all.end(), chunk_hc.begin(), chunk_hc.end()); - if (out_logits) { + if (chunk_output) { if (is_last_shard) { last_out = std::move(chunk_out); } else { @@ -7067,13 +7278,12 @@ bool deepseek4_step_layer_range( Ds4VerifyHooks * fused_graph_hooks = (fused_hybrid_decode && !verify_hooks) ? &fused_hybrid_decode_hooks : verify_hooks; - if ((!moe_hybrid || fused_hybrid_ready) && - ((n_tokens >= 2 && - (n_tokens <= DS4_CONSERVATIVE_VERIFY_MAX_TOKENS || - wide_verify_candidate) && verify_hooks) || - fused_hybrid_decode) && - layer_begin == 0 && is_last_shard && - out_logits && ds4_backend_is_gpu(backend) && ds4_fused_verify_enabled()) { + const bool fused_hybrid_decode_candidate = + deepseek4_should_attempt_fused_hybrid_decode( + fused_hybrid_decode, layer_begin == 0 && is_last_shard, + execute_output_path, ds4_backend_is_gpu(backend), + ds4_fused_verify_enabled()); + if (fused_verify_candidate || fused_hybrid_decode_candidate) { const bool q1_feature_capture = n_tokens == 1 && verify_hooks && verify_hooks->capture_out; // q=1 target-feature capture walks many prompt-position shapes. Keep @@ -7086,7 +7296,7 @@ bool deepseek4_step_layer_range( graph_cache, q1_feature_capture, fused_decode_graph_cache, backend, w, cache, hc_layer_weights_range, hc_output_weights_range, hash_routing_tables_range, - scratch.hash_expert_ids, embed, n_tokens, kv_start, *out_logits, token_ids, + scratch.hash_expert_ids, embed, n_tokens, kv_start, out_logits, token_ids, fused_graph_hooks, telemetry, fused_hybrid_ready ? moe_hybrid : nullptr, routing_stats); if (vrc < 0) return false; @@ -7119,12 +7329,12 @@ bool deepseek4_step_layer_range( if (!moe_hybrid && n_tokens == 1 && allow_decode_graph_reuse && layer_begin == 0 && is_last_shard && !(verify_hooks && verify_hooks->capture_layer_ids && verify_hooks->capture_out) && - out_logits && ds4_backend_is_gpu(backend) && + execute_output_path && ds4_backend_is_gpu(backend) && ds4_fused_decode_enabled(w)) { const int rc = ds4_try_fused_decode_step( fused_decode_graph_cache, backend, w, cache, hc_layer_weights_range, hc_output_weights_range, hash_routing_tables_range, scratch.hash_expert_ids, - embed, kv_start, *out_logits, token_ids, telemetry); + embed, kv_start, out_logits, token_ids, telemetry); if (rc < 0) return false; if (rc > 0) { const int np = kv_start + 1; @@ -7357,9 +7567,15 @@ bool deepseek4_step_layer_range( attn_post_backend = cached.post; attn_comb_backend = cached.comb; } else { - hc_pre_batch(cur, hc_post, hc_comb, - hc_state.data(), hc_lw.attn, L.hc_attn_fn, - n_tokens, n_embd, n_hc, w.n_hc_sinkhorn_iter, w.hc_eps); + if (!hc_pre_batch(cur, hc_post, hc_comb, + hc_state.data(), hc_lw.attn, L.hc_attn_fn, + device, n_tokens, n_embd, n_hc, + w.n_hc_sinkhorn_iter, w.hc_eps)) { + std::fprintf(stderr, + "[deepseek4] HC-pre device selection failed " + "layer %d attn\n", il); + return false; + } } if (telemetry) telemetry->hc_pre_attn_us += ds4_elapsed_us(hc_pre_attn_t0, Ds4TimingClock::now()); @@ -7382,7 +7598,8 @@ bool deepseek4_step_layer_range( : DeepSeek4AttentionImpl::Explicit; if (!ds4_run_exact_tokenwise_prefill_attention( backend, w, L, lc, il, cur.data(), n_tokens, kv_start, - attention_impl, attn_out_host, + attention_impl, exact_prefill_stable_raw_order, + attn_out_host, cached_attn_allocs[(size_t) il], telemetry)) { return false; } @@ -7795,9 +8012,15 @@ bool deepseek4_step_layer_range( ffn_post_backend = cached.post; ffn_comb_backend = cached.comb; } else { - hc_pre_batch(ffn_working, hc_post, hc_comb, - hc_state.data(), hc_lw.ffn, L.hc_ffn_fn, - n_tokens, n_embd, n_hc, w.n_hc_sinkhorn_iter, w.hc_eps); + if (!hc_pre_batch(ffn_working, hc_post, hc_comb, + hc_state.data(), hc_lw.ffn, L.hc_ffn_fn, + device, n_tokens, n_embd, n_hc, + w.n_hc_sinkhorn_iter, w.hc_eps)) { + std::fprintf(stderr, + "[deepseek4] HC-pre device selection failed " + "layer %d ffn\n", il); + return false; + } } if (telemetry) telemetry->hc_pre_ffn_us += ds4_elapsed_us(hc_pre_ffn_t0, Ds4TimingClock::now()); @@ -7843,6 +8066,7 @@ bool deepseek4_step_layer_range( token_ids, hash_routing_tables_range[(size_t)il], *moe_hybrid, expert_runtime, routing_stats, ffn_out_host, telemetry, + exact_prefill_q1_ffn_order, ffn_device_join ? &owner_outputs : nullptr)) { std::fprintf(stderr, "[deepseek4-moe-tp] layer-range FFN failed layer %d\n", @@ -8009,7 +8233,7 @@ bool deepseek4_step_layer_range( } // ── Output: HC pre → norm → lm_head (or return hidden state) ──────── - if (is_last_shard && out_logits) { + if (is_last_shard && execute_output_path) { // Final HC pre for output const auto output_t0 = Ds4TimingClock::now(); std::vector & final_embd = scratch.final_embd; @@ -8035,9 +8259,13 @@ bool deepseek4_step_layer_range( if (ggml_backend_graph_compute(backend, cached_decode_output_graph.sg.gf) != GGML_STATUS_SUCCESS) { return false; } - out_logits->resize((size_t)w.n_vocab); - ggml_backend_tensor_get(cached_decode_output_graph.sg.logits, - out_logits->data(), 0, sizeof(float) * (size_t)w.n_vocab); + if (readback_logits) { + out_logits->resize((size_t)w.n_vocab); + ggml_backend_tensor_get( + cached_decode_output_graph.sg.logits, + out_logits->data(), 0, + sizeof(float) * (size_t)w.n_vocab); + } } else { const size_t ctx_size = 16 * 1024 * 1024; ggml_init_params params{}; @@ -8083,11 +8311,14 @@ bool deepseek4_step_layer_range( return false; } - out_logits->resize((size_t)w.n_vocab); - const size_t logits_offset = last_only ? 0 : - (size_t)(n_tokens - 1) * (size_t)w.n_vocab * sizeof(float); - ggml_backend_tensor_get(logits, out_logits->data(), logits_offset, - sizeof(float) * (size_t)w.n_vocab); + if (readback_logits) { + out_logits->resize((size_t)w.n_vocab); + const size_t logits_offset = last_only ? 0 : + (size_t)(n_tokens - 1) * (size_t)w.n_vocab * sizeof(float); + ggml_backend_tensor_get( + logits, out_logits->data(), logits_offset, + sizeof(float) * (size_t)w.n_vocab); + } if (verify_hooks && verify_hooks->all_logits_out) { verify_hooks->all_logits_out->resize((size_t) w.n_vocab * n_tokens); ggml_backend_tensor_get(logits, verify_hooks->all_logits_out->data(), 0, diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index eea667423..b01c3ff5c 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -343,6 +343,20 @@ int deepseek4_previous_raw_ring_spans( int kv_start, int n_swa, DeepSeek4RawRingSpan spans[2]); +bool deepseek4_exact_tokenwise_uses_runtime_raw_row( + bool exact_prefill_stable_raw_order, + int token_position, + int n_swa); + +// Exact q=4 prefill must retain the q<=3 owner-kernel reduction order. The +// four-row dual-owner hybrid FFN path is numerically different from q=1 on the +// production target. +// Return the largest owner sub-batch allowed for this production intent. +int deepseek4_exact_prefill_hybrid_ffn_sub_batch( + bool exact_prefill_q1_ffn_order, + int n_tokens); +bool deepseek4_exact_prefill_route_graph_requires_eager( + bool exact_prefill_q1_ffn_order); bool deepseek4_snapshot_save(const DeepSeek4Cache & cache, ggml_backend_t snapshot_backend, DeepSeek4Snapshot & out); @@ -356,6 +370,50 @@ int deepseek4_safe_compressor_batch_tokens(const DeepSeek4Weights & w, int kv_start, int n_tokens); +int deepseek4_prefill_chunk_tokens(PrefillAttentionMode mode, + bool exact_bands_enabled, + bool batch_supported, + int requested_chunk, + int layer_major_cap); + +bool deepseek4_env_flag_value_enabled(const char * value); + +struct DeepSeek4PrefillOutputIntent { + bool execute_output_path = false; + bool readback_logits = false; +}; + +DeepSeek4PrefillOutputIntent deepseek4_prefill_output_intent( + PrefillAttentionMode mode, + bool exact_bands_active, + int n_tokens, + bool is_final_chunk, + bool ends_at_snapshot, + bool external_requires_logits); + +DeepSeek4PrefillOutputIntent deepseek4_prepare_prefill_output_intent( + PrefillAttentionMode mode, + bool exact_bands_active, + int n_tokens, + bool is_final_chunk, + bool ends_at_snapshot, + bool external_requires_logits, + std::vector & last_logits, + int & last_logits_pos); + +void deepseek4_invalidate_prefill_logits_if_skipped( + bool readback_logits, + std::vector & last_logits, + int & last_logits_pos); + +bool deepseek4_commit_prefill_logits( + bool readback_logits, + int vocab_size, + int cache_position, + std::vector && logits, + std::vector & last_logits, + int & last_logits_pos); + // Forward: single step (prefill chunk or decode token). // embed: [n_embd, n_tokens] input embeddings (post-embedding lookup). // hc_state: [n_hc * n_embd] persistent HC residual (updated in-place). @@ -393,8 +451,63 @@ struct Ds4VerifyHooks { std::vector * all_logits_out = nullptr; // [n_vocab * n_tokens] std::vector * argmax_out = nullptr; // [n_tokens], optional GPU result bool prefer_argmax_only = false; // skip logits D2H when available + // Prefill uses the capture fields too, but must never enter the + // intentionally approximate fused-verification path. + bool allow_fused_verify = true; }; +Ds4VerifyHooks deepseek4_make_prefill_capture_hooks( + const std::vector * capture_layer_ids, + std::vector * capture_out, + int capture_token_begin, + int capture_token_end); + +bool deepseek4_should_attempt_fused_verify( + int n_tokens, + const Ds4VerifyHooks * verify_hooks, + bool owner_topology_supported, + bool full_layer_range, + bool execute_output_path, + bool gpu_backend, + bool fused_verify_enabled); + +bool deepseek4_should_attempt_wide_fused_verify( + int n_tokens, + const Ds4VerifyHooks * verify_hooks, + bool q5_enabled, + bool owner_topology_supported, + bool full_layer_range, + bool has_output_storage, + bool gpu_backend, + bool fused_verify_enabled); + +bool deepseek4_should_warn_fused_verify_inactive( + int n_tokens, + const Ds4VerifyHooks * verify_hooks, + bool full_layer_range, + bool fused_verify_enabled, + bool fused_verify_candidate); + +struct DeepSeek4RecursiveOutputIntent { + bool execute_output_path = false; + bool pass_output_storage = false; +}; + +DeepSeek4RecursiveOutputIntent deepseek4_recursive_output_intent( + PrefillAttentionMode mode, + bool parent_execute_output_path, + bool parent_has_output_storage, + bool is_last_shard, + int chunk_tokens, + bool is_final_chunk); + +bool deepseek4_should_attempt_fused_hybrid_decode( + bool fused_hybrid_decode, + bool full_layer_range, + bool execute_output_path, + bool gpu_backend, + bool fused_verify_enabled); + bool deepseek4_step_layer_range( ggml_backend_t backend, int device, @@ -413,7 +526,10 @@ bool deepseek4_step_layer_range( Ds4VerifyHooks * verify_hooks = nullptr, MoeHybridStorage * moe_hybrid = nullptr, MoeExpertComputeRuntime * expert_runtime = nullptr, - MoeHybridRoutingStats * routing_stats = nullptr); + MoeHybridRoutingStats * routing_stats = nullptr, + bool execute_output_path = false, + bool exact_prefill_stable_raw_order = false, + bool exact_prefill_q1_ffn_order = false); bool build_deepseek4_moe_hybrid_storage_from_file( const std::string & path, diff --git a/server/tests/test_deepseek4_unit.cpp b/server/tests/test_deepseek4_unit.cpp index 4a2b5f448..1a6b86dbe 100644 --- a/server/tests/test_deepseek4_unit.cpp +++ b/server/tests/test_deepseek4_unit.cpp @@ -1178,6 +1178,30 @@ static void test_raw_ring_spans_after_wrap() { TEST_ASSERT(spans[1].count == 2); TEST_ASSERT(spans[0].count + spans[1].count == 7); + TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(true, -1, 8)); + TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(true, 6, 8)); + TEST_ASSERT(deepseek4_exact_tokenwise_uses_runtime_raw_row(true, 7, 8)); + TEST_ASSERT(deepseek4_exact_tokenwise_uses_runtime_raw_row(true, 8, 8)); + TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(true, 8, 0)); + // Default dynamic speculative verification is exact multi-token work too, + // but it must preserve its historical chronological-span topology. + TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(false, 7, 8)); + TEST_ASSERT(!deepseek4_exact_tokenwise_uses_runtime_raw_row(false, 8, 8)); + + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); +} + +static void test_exact_prefill_hybrid_ffn_sub_batch() { + std::fprintf(stderr, " test_exact_prefill_hybrid_ffn_sub_batch ..."); + // Exact q4 keeps prompt geometry but evaluates independent hybrid-FFN + // owner rows as q3+q1. Every non-exact caller retains its prior q4 batch. + TEST_ASSERT(deepseek4_exact_prefill_hybrid_ffn_sub_batch(true, 4) == 3); + TEST_ASSERT(deepseek4_exact_prefill_hybrid_ffn_sub_batch(true, 3) == 3); + TEST_ASSERT(deepseek4_exact_prefill_hybrid_ffn_sub_batch(true, 1) == 1); + TEST_ASSERT(deepseek4_exact_prefill_hybrid_ffn_sub_batch(false, 4) == 4); + TEST_ASSERT(deepseek4_exact_prefill_hybrid_ffn_sub_batch(false, 5) == 5); + TEST_ASSERT(deepseek4_exact_prefill_route_graph_requires_eager(true)); + TEST_ASSERT(!deepseek4_exact_prefill_route_graph_requires_eager(false)); std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } @@ -1468,6 +1492,8 @@ static void test_backend_sampling_penalizes_prompt_history() { auto decode_one = [&](const SamplerCfg & sampler) { backend.last_logits_ = {0.0f, 4.0f, 3.0f}; + backend.last_logits_pos_ = 1; + backend.cache_.cur_pos = 1; backend.sampler_ = sampler; emitted.clear(); std::vector generated; @@ -1489,6 +1515,24 @@ static void test_backend_sampling_penalizes_prompt_history() { penalized.rep_pen = 2.0f; TEST_ASSERT(decode_one(penalized) == 2); + backend.last_logits_ = {0.0f, 4.0f, 3.0f}; + backend.last_logits_pos_ = 0; + backend.cache_.cur_pos = 1; + std::vector generated; + emitted.clear(); + TEST_ASSERT(!backend.do_decode( + /*committed=*/1, /*n_gen=*/1, /*history_prefix=*/{1}, + generated, io)); + TEST_ASSERT(emitted.empty()); + + backend.last_logits_.clear(); + backend.last_logits_pos_ = -1; + generated.clear(); + TEST_ASSERT(!backend.do_decode( + /*committed=*/1, /*n_gen=*/1, /*history_prefix=*/{1}, + generated, io)); + TEST_ASSERT(emitted.empty()); + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } @@ -1682,6 +1726,335 @@ static void test_safe_compressor_batch_tokens() { std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } +static void test_exact_prefill_chunk_policy() { + std::fprintf(stderr, " test_exact_prefill_chunk_policy ..."); + + TEST_ASSERT(!deepseek4_env_flag_value_enabled(nullptr)); + TEST_ASSERT(!deepseek4_env_flag_value_enabled("")); + TEST_ASSERT(!deepseek4_env_flag_value_enabled("0")); + TEST_ASSERT(deepseek4_env_flag_value_enabled("1")); + + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Exact, /*exact_bands_enabled=*/false, + /*batch_supported=*/true, /*requested_chunk=*/4, + /*layer_major_cap=*/512) == 1); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Exact, /*exact_bands_enabled=*/true, + /*batch_supported=*/true, /*requested_chunk=*/1, + /*layer_major_cap=*/512) == 1); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Exact, true, true, 2, 512) == 2); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Exact, true, true, 3, 512) == 3); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Exact, true, true, 4, 512) == 4); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Exact, true, true, 8, 512) == 4); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Exact, true, + /*batch_supported=*/false, 4, 512) == 1); + + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Dense, false, + /*batch_supported=*/false, 8, 512) == 1); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Sparse, false, + /*batch_supported=*/true, 8, 512) == 8); + TEST_ASSERT(deepseek4_prefill_chunk_tokens( + PrefillAttentionMode::Sparse, false, + /*batch_supported=*/true, 1024, 512) == 512); + + TEST_ASSERT(deepseek4_prefill_allows_decode_graph_reuse(false)); + TEST_ASSERT(!deepseek4_prefill_allows_decode_graph_reuse(true)); + + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); +} + +static void test_exact_prefill_band_schedule() { + std::fprintf(stderr, " test_exact_prefill_band_schedule ..."); + DeepSeek4Weights w; + w.compress_ratios = {4, 128}; + + for (int width = 1; width <= 4; ++width) { + for (int start = 0; start <= 5; ++start) { + for (int total = 1; total <= 9; ++total) { + int consumed = 0; + while (consumed < total) { + int outer = std::min(width, total - consumed); + int inner = 0; + while (inner < outer) { + const int pos = start + consumed + inner; + const int step = deepseek4_safe_compressor_batch_tokens( + w, pos, outer - inner); + TEST_ASSERT(step >= 1 && step <= width); + TEST_ASSERT((pos % 4) + step <= 4); + inner += step; + } + TEST_ASSERT(inner == outer); + consumed += outer; + } + TEST_ASSERT(consumed == total); + } + } + } + + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); +} + +static void test_prefill_output_intents() { + std::fprintf(stderr, " test_prefill_output_intents ..."); + + // Legacy/default exact q1 and explicit --chunk 1 retain both the + // established output topology and their historical host readback. + const auto legacy_q1 = deepseek4_prefill_output_intent( + PrefillAttentionMode::Exact, /*exact_bands_active=*/false, + /*n_tokens=*/1, /*is_final_chunk=*/false, + /*ends_at_snapshot=*/false, /*external_requires_logits=*/false); + TEST_ASSERT(legacy_q1.execute_output_path); + TEST_ASSERT(legacy_q1.readback_logits); + + // Dense and sparse policies are outside exact-band readout elision. + for (PrefillAttentionMode mode : + {PrefillAttentionMode::Dense, PrefillAttentionMode::Sparse}) { + const auto intent = deepseek4_prefill_output_intent( + mode, false, 4, false, false, false); + TEST_ASSERT(intent.execute_output_path); + TEST_ASSERT(intent.readback_logits); + } + + // Interior singleton leaves created by enabled q2/q3/q4 exact prefill + // preserve the q1/fused execution topology without a host logits readback. + DeepSeek4Weights failure_weights; + failure_weights.compress_ratios = {4, 128}; + for (int width : {2, 3, 4}) { + TEST_ASSERT(deepseek4_safe_compressor_batch_tokens( + failure_weights, /*kv_start=*/2763, width) == 1); + const auto fallback = deepseek4_prefill_output_intent( + PrefillAttentionMode::Exact, /*exact_bands_active=*/true, + /*n_tokens=*/1, /*is_final_chunk=*/false, + /*ends_at_snapshot=*/false, + /*external_requires_logits=*/false); + TEST_ASSERT_MSG(fallback.execute_output_path, + "exact singleton must preserve output topology"); + TEST_ASSERT_MSG(!fallback.readback_logits, + "exact singleton must skip host logits readback"); + } + + // A capture-only singleton has the same topology requirement, but capture + // values are not an external vocabulary-logits consumer. + const auto capture_only = deepseek4_prefill_output_intent( + PrefillAttentionMode::Exact, true, 1, false, false, false); + TEST_ASSERT(capture_only.execute_output_path); + TEST_ASSERT(!capture_only.readback_logits); + + // Final and exact snapshot endpoints still transfer current logits. + const auto final_singleton = deepseek4_prefill_output_intent( + PrefillAttentionMode::Exact, true, 1, true, false, false); + TEST_ASSERT(final_singleton.execute_output_path); + TEST_ASSERT(final_singleton.readback_logits); + const auto snapshot_singleton = deepseek4_prefill_output_intent( + PrefillAttentionMode::Exact, true, 1, false, true, false); + TEST_ASSERT(snapshot_singleton.execute_output_path); + TEST_ASSERT(snapshot_singleton.readback_logits); + + // A genuine external vocabulary consumer is independently authoritative. + const auto external = deepseek4_prefill_output_intent( + PrefillAttentionMode::Exact, true, 4, false, false, true); + TEST_ASSERT(external.execute_output_path); + TEST_ASSERT(external.readback_logits); + + // Geometry from the hardware failure: the 128-token capture window begins + // at 2891 - 128 == 2763. A q=2 band beginning at 2762 is split into an + // interior singleton ending at 2763, which must not read back logits. + constexpr int prompt_tokens = 2891; + constexpr int capture_begin = prompt_tokens - 128; + TEST_ASSERT(capture_begin == 2763); + TEST_ASSERT(DeepSeek4Backend::capture_safe_prefill_tokens( + /*token_offset=*/2762, /*requested_tokens=*/2, + /*final_capture_from=*/capture_begin, + /*batch_final_capture=*/false, + /*snapshot_pending=*/false, + /*snapshot_capture_from=*/0, + /*snapshot_capture_to=*/0) == 1); + TEST_ASSERT(deepseek4_safe_compressor_batch_tokens( + failure_weights, capture_begin, /*n_tokens=*/2) == 1); + const auto failure_geometry = deepseek4_prefill_output_intent( + PrefillAttentionMode::Exact, true, 1, + /*is_final_chunk=*/false, /*ends_at_snapshot=*/false, + /*external_requires_logits=*/false); + TEST_ASSERT(failure_geometry.execute_output_path); + TEST_ASSERT(!failure_geometry.readback_logits); + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); +} + +static void test_prefill_readout_lifecycle_and_fused_exclusion() { + std::fprintf(stderr, + " test_prefill_readout_lifecycle_and_fused_exclusion ..."); + + std::vector last_logits = {1.0f, 2.0f}; + int last_logits_pos = 7; + deepseek4_invalidate_prefill_logits_if_skipped( + /*readback_logits=*/true, last_logits, last_logits_pos); + TEST_ASSERT(last_logits == std::vector({1.0f, 2.0f})); + TEST_ASSERT(last_logits_pos == 7); + const auto prepared_no_readback = deepseek4_prepare_prefill_output_intent( + PrefillAttentionMode::Exact, /*exact_bands_active=*/true, + /*n_tokens=*/1, /*is_final_chunk=*/false, + /*ends_at_snapshot=*/false, /*external_requires_logits=*/false, + last_logits, last_logits_pos); + TEST_ASSERT(prepared_no_readback.execute_output_path); + TEST_ASSERT(!prepared_no_readback.readback_logits); + TEST_ASSERT(last_logits.empty()); + TEST_ASSERT(last_logits_pos == -1); + + // The production recursive selector scopes host-readback suppression to + // exact mode. Dense and sparse retain their legacy per-subchunk output. + for (PrefillAttentionMode mode : + {PrefillAttentionMode::Dense, PrefillAttentionMode::Sparse}) { + for (int width : {1, 2, 3, 4}) { + const auto interior = deepseek4_recursive_output_intent( + mode, /*parent_execute_output_path=*/true, + /*parent_has_output_storage=*/true, + /*is_last_shard=*/true, /*chunk_tokens=*/width, + /*is_final_chunk=*/false); + TEST_ASSERT(interior.execute_output_path); + TEST_ASSERT(interior.pass_output_storage); + } + } + const auto exact_interior = deepseek4_recursive_output_intent( + PrefillAttentionMode::Exact, + /*parent_execute_output_path=*/false, + /*parent_has_output_storage=*/false, + /*is_last_shard=*/true, /*chunk_tokens=*/1, + /*is_final_chunk=*/false); + TEST_ASSERT(exact_interior.execute_output_path); + TEST_ASSERT(!exact_interior.pass_output_storage); + const auto exact_final = deepseek4_recursive_output_intent( + PrefillAttentionMode::Exact, + /*parent_execute_output_path=*/true, + /*parent_has_output_storage=*/true, + /*is_last_shard=*/true, /*chunk_tokens=*/1, + /*is_final_chunk=*/true); + TEST_ASSERT(exact_final.execute_output_path); + TEST_ASSERT(exact_final.pass_output_storage); + + // Final/snapshot readbacks must be current, vocabulary-sized values. + constexpr int vocab_size = 4; + TEST_ASSERT(deepseek4_commit_prefill_logits( + /*readback_logits=*/true, vocab_size, /*cache_position=*/2764, + std::vector{1.0f, 2.0f, 3.0f, 4.0f}, + last_logits, last_logits_pos)); + TEST_ASSERT(last_logits.size() == vocab_size); + TEST_ASSERT(last_logits_pos == 2764); + TEST_ASSERT(deepseek4_commit_prefill_logits( + /*readback_logits=*/true, vocab_size, /*cache_position=*/2891, + std::vector{4.0f, 3.0f, 2.0f, 1.0f}, + last_logits, last_logits_pos)); + TEST_ASSERT(last_logits.size() == vocab_size); + TEST_ASSERT(last_logits.front() == 4.0f); + TEST_ASSERT(last_logits_pos == 2891); + + // A malformed readback cannot leave previously current logits visible. + TEST_ASSERT(!deepseek4_commit_prefill_logits( + /*readback_logits=*/true, vocab_size, /*cache_position=*/3000, + std::vector{9.0f}, last_logits, last_logits_pos)); + TEST_ASSERT(last_logits.empty()); + TEST_ASSERT(last_logits_pos == -1); + + std::vector capture_ids = {1, 3}; + std::vector capture; + Ds4VerifyHooks prefill_hooks = deepseek4_make_prefill_capture_hooks( + &capture_ids, &capture, /*capture_token_begin=*/1, + /*capture_token_end=*/3); + TEST_ASSERT(prefill_hooks.capture_layer_ids == &capture_ids); + TEST_ASSERT(prefill_hooks.capture_out == &capture); + TEST_ASSERT(prefill_hooks.capture_token_begin == 1); + TEST_ASSERT(prefill_hooks.capture_token_end == 3); + TEST_ASSERT(!prefill_hooks.allow_fused_verify); + TEST_ASSERT(deepseek4_should_attempt_fused_hybrid_decode( + /*fused_hybrid_decode=*/true, /*full_layer_range=*/true, + /*execute_output_path=*/prepared_no_readback.execute_output_path, + /*gpu_backend=*/true, /*fused_verify_enabled=*/true)); + TEST_ASSERT(!deepseek4_should_attempt_fused_hybrid_decode( + /*fused_hybrid_decode=*/true, /*full_layer_range=*/true, + /*execute_output_path=*/false, + /*gpu_backend=*/true, /*fused_verify_enabled=*/true)); + TEST_ASSERT(!deepseek4_should_attempt_fused_verify( + /*n_tokens=*/4, &prefill_hooks, + /*owner_topology_supported=*/true, + /*full_layer_range=*/true, + /*has_logits_output=*/true, + /*gpu_backend=*/true, + /*fused_verify_enabled=*/true)); + TEST_ASSERT(!deepseek4_should_warn_fused_verify_inactive( + /*n_tokens=*/4, &prefill_hooks, + /*full_layer_range=*/true, + /*fused_verify_enabled=*/true, + /*fused_verify_candidate=*/false)); + + Ds4VerifyHooks verifier_hooks; + TEST_ASSERT(verifier_hooks.allow_fused_verify); + TEST_ASSERT(deepseek4_should_attempt_fused_verify( + /*n_tokens=*/DS4_CONSERVATIVE_VERIFY_MAX_TOKENS, &verifier_hooks, + /*owner_topology_supported=*/true, + /*full_layer_range=*/true, + /*has_logits_output=*/true, + /*gpu_backend=*/true, + /*fused_verify_enabled=*/true)); + TEST_ASSERT(!deepseek4_should_attempt_fused_verify( + /*n_tokens=*/1, &verifier_hooks, true, true, true, true, true)); + TEST_ASSERT(!deepseek4_should_attempt_fused_verify( + /*n_tokens=*/DS4_Q5_VERIFY_TOKENS, + &verifier_hooks, true, true, true, true, true)); + TEST_ASSERT(!deepseek4_should_attempt_fused_verify( + /*n_tokens=*/DS4_Q5_VERIFY_TOKENS + 1, + &verifier_hooks, true, true, true, true, true)); + TEST_ASSERT(!deepseek4_should_attempt_wide_fused_verify( + DS4_Q5_VERIFY_TOKENS, &verifier_hooks, + /*q5_enabled=*/false, true, true, true, true, true)); + TEST_ASSERT(deepseek4_should_attempt_wide_fused_verify( + DS4_Q5_VERIFY_TOKENS, &verifier_hooks, + /*q5_enabled=*/true, true, true, true, true, true)); + TEST_ASSERT(!deepseek4_should_attempt_wide_fused_verify( + DS4_Q5_VERIFY_TOKENS, &verifier_hooks, + /*q5_enabled=*/true, true, true, + /*has_output_storage=*/false, true, true)); + TEST_ASSERT(!deepseek4_should_attempt_wide_fused_verify( + DS4_CONSERVATIVE_VERIFY_MAX_TOKENS, &verifier_hooks, + /*q5_enabled=*/true, true, true, true, true, true)); + TEST_ASSERT(deepseek4_should_warn_fused_verify_inactive( + /*n_tokens=*/4, &verifier_hooks, + /*full_layer_range=*/true, + /*fused_verify_enabled=*/true, + /*fused_verify_candidate=*/false)); + TEST_ASSERT(!deepseek4_should_warn_fused_verify_inactive( + /*n_tokens=*/4, /*verify_hooks=*/nullptr, + /*full_layer_range=*/true, + /*fused_verify_enabled=*/true, + /*fused_verify_candidate=*/false)); + TEST_ASSERT(!deepseek4_should_warn_fused_verify_inactive( + /*n_tokens=*/4, &verifier_hooks, + /*full_layer_range=*/true, + /*fused_verify_enabled=*/true, + /*fused_verify_candidate=*/true)); + TEST_ASSERT(!deepseek4_should_warn_fused_verify_inactive( + /*n_tokens=*/1, &verifier_hooks, + /*full_layer_range=*/true, + /*fused_verify_enabled=*/true, + /*fused_verify_candidate=*/false)); + TEST_ASSERT(!deepseek4_should_warn_fused_verify_inactive( + /*n_tokens=*/4, &verifier_hooks, + /*full_layer_range=*/false, + /*fused_verify_enabled=*/true, + /*fused_verify_candidate=*/false)); + TEST_ASSERT(!deepseek4_should_warn_fused_verify_inactive( + /*n_tokens=*/4, &verifier_hooks, + /*full_layer_range=*/true, + /*fused_verify_enabled=*/false, + /*fused_verify_candidate=*/false)); + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); +} + static void test_hybrid_prefill_chunk_tokens() { std::fprintf(stderr, " test_hybrid_prefill_chunk_tokens ..."); TEST_ASSERT(deepseek4_hybrid_prefill_chunk_tokens(2048, 0) == 2048); @@ -4111,6 +4484,7 @@ int main() { test_indexer_mask_cpu(backend); test_hash_routing_lookup(); test_raw_ring_spans_after_wrap(); + test_exact_prefill_hybrid_ffn_sub_batch(); test_auto_split_computation(); test_layer_range_validation(); test_hc_state_dimensions(); @@ -4127,6 +4501,10 @@ int main() { test_dspark_loader_contract_and_bounds(backend); test_dspark_confidence_uses_separate_hidden(backend); test_safe_compressor_batch_tokens(); + test_exact_prefill_chunk_policy(); + test_exact_prefill_band_schedule(); + test_prefill_output_intents(); + test_prefill_readout_lifecycle_and_fused_exclusion(); test_hybrid_prefill_chunk_tokens(); test_dspark_park_all_releases_drafter(); test_dspark_raw_ring_rollback_after_wrap(backend);