diff --git a/server/docs/DS4.md b/server/docs/DS4.md index 3de39b571..350958f8c 100644 --- a/server/docs/DS4.md +++ b/server/docs/DS4.md @@ -343,9 +343,18 @@ export DFLASH_DS4_SPEC_Q=4 ./server/build-hip/dflash_server /path/to/deepseek4-target.gguf \ --target-device hip:0 \ + --ds4-fused-verify-f16-kv \ --ds4-fused-decode ``` +`--ds4-fused-verify-f16-kv` feeds the persistent F16 MLA cache directly to +batched explicit verifier attention instead of converting the full cache to +F32 on every speculative step. Key-side accumulation remains F32 through 512 +attention rows to preserve the short-context quality baseline. The option is +currently qualified only for a single HIP target and remains off by default. +It changes verifier floating-point inputs and can change generated tokens, so +re-run workload quality checks before enabling it for another checkpoint. + `DFLASH_DS4_FUSED_VERIFY=1` is the opt-in throughput profile. Its persistent whole-model GPU graph uses stable padded reduction shapes, so near-tied greedy logits can select a different token than the normal causal verifier even at diff --git a/server/src/common/backend_args.h b/server/src/common/backend_args.h index ee7b55d33..7bf8dadf5 100644 --- a/server/src/common/backend_args.h +++ b/server/src/common/backend_args.h @@ -59,6 +59,7 @@ struct BackendArgs { // deepseek4-specific decode options int ds4_expert_top_k = 0; // 0 = model default bool ds4_fused_decode = false; + bool ds4_fused_verify_f16_kv = false; // Attention and speculative-decode options. Individual backends consume // only the fields they support. diff --git a/server/src/common/backend_factory.cpp b/server/src/common/backend_factory.cpp index 0d1ccc61b..0b1a8a2db 100644 --- a/server/src/common/backend_factory.cpp +++ b/server/src/common/backend_factory.cpp @@ -424,6 +424,7 @@ std::unique_ptr create_backend( cfg.chunk = args.chunk; cfg.expert_top_k = args.ds4_expert_top_k; cfg.fused_decode = args.ds4_fused_decode; + cfg.fused_verify_f16_kv = args.ds4_fused_verify_f16_kv; cfg.prefill_mode = args.ds4_prefill_mode; auto backend = std::make_unique(cfg); diff --git a/server/src/common/feature_gate.cpp b/server/src/common/feature_gate.cpp index f655f5d4f..0bcb2f1bb 100644 --- a/server/src/common/feature_gate.cpp +++ b/server/src/common/feature_gate.cpp @@ -280,6 +280,12 @@ std::string check_feature_compatibility( "DeepSeek4"; } + // ── --ds4-fused-verify-f16-kv × placement + if (args.ds4_fused_verify_f16_kv && !monolithic_ds4) { + return "--ds4-fused-verify-f16-kv currently requires single-device " + "HIP DeepSeek4"; + } + // ── --ds4-expert-top-k × architecture/adapter if (args.ds4_expert_top_k != 0 && !local_ds4) { return "--ds4-expert-top-k currently requires a single local " diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index 7183c4876..a9fdb4bf3 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -674,7 +674,7 @@ DeepSeek4Backend::~DeepSeek4Backend() { } bool DeepSeek4Backend::requires_monolithic_model() const { - return cfg_.fused_decode || + return cfg_.fused_decode || cfg_.fused_verify_f16_kv || prefill_attention_mode_is_approximate(cfg_.prefill_mode); } @@ -719,9 +719,11 @@ bool DeepSeek4Backend::load_model() { (force_full || need_monolithic)) { std::fprintf(stderr, "[deepseek4] monolithic execution requested " - "(forced=%s, fused_decode=%s, prefill=%s)\n", + "(forced=%s, fused_decode=%s, " + "fused_verify_f16_kv=%s, prefill=%s)\n", force_full ? "yes" : "no", cfg_.fused_decode ? "on" : "off", + cfg_.fused_verify_f16_kv ? "on" : "off", prefill_attention_mode_name(cfg_.prefill_mode)); if (!load_deepseek4_gguf(cfg_.model_path, backend_, w_)) { if (prefill_attention_mode_is_approximate(cfg_.prefill_mode)) { @@ -758,11 +760,17 @@ bool DeepSeek4Backend::load_model() { } w_.routed_expert_top_k = cfg_.expert_top_k; w_.fused_decode = cfg_.fused_decode && !moe_hybrid_; + w_.fused_verify_f16_kv = cfg_.fused_verify_f16_kv && !moe_hybrid_; if (cfg_.fused_decode && moe_hybrid_) { std::fprintf(stderr, "[deepseek4] fused decode unavailable with hybrid expert placement; " "using layered decode\n"); } + if (cfg_.fused_verify_f16_kv && moe_hybrid_) { + std::fprintf(stderr, + "[deepseek4] fused verifier F16 K/V unavailable with hybrid " + "expert placement; using F32 verifier attention\n"); + } return true; } diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index f9c0c00b5..a965e6906 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -60,6 +60,11 @@ static bool ds4_env_flag(const char * name) { return value && value[0] && std::strcmp(value, "0") != 0; } +// F32 key/value-side accumulation protects the short-context quality baseline +// while still avoiding a full-cache F16 -> F32 conversion once attention is +// large enough for that conversion to dominate verifier time. +static constexpr int DS4_FUSED_VERIFY_F16_F32_KV_MAX_ATTN = 512; + static int ds4_effective_expert_count(const DeepSeek4Weights & w) { int requested = w.routed_expert_top_k; if (const char * value = std::getenv("DFLASH_DS4_TOPK")) { @@ -1701,6 +1706,7 @@ static ggml_tensor * build_mla_attention( const bool layer_major_batch = causal_batch && attention_impl != DeepSeek4AttentionImpl::Explicit; ggml_tensor * old_rows_scratch = nullptr; + ggml_tensor * old_rows_scratch_f16 = nullptr; int n_old_rows = 0; ggml_tensor * prior_rows_scratch = nullptr; int n_prior_rows = 0; @@ -1718,6 +1724,7 @@ static ggml_tensor * build_mla_attention( ? ggml_concat(ctx, old_rows_scratch, saved, 1) : saved; n_old_rows++; } + old_rows_scratch_f16 = old_rows_scratch; old_rows_scratch = ds4_cast_if_needed(ctx, old_rows_scratch, GGML_TYPE_F32); } else if (causal_batch && !layer_major_batch) { // Copy the to-be-overwritten rows FIRST; same-stream build order runs @@ -1947,13 +1954,51 @@ static ggml_tensor * build_mla_attention( } else { kv_attn = raw_kv_view(0, n_raw); } - if (n_comp_attn > 0 && comp_kv_source) { - ggml_tensor * comp = ggml_view_2d(ctx, comp_kv_source, head_dim, n_comp_attn, comp_kv_source->nb[1], 0); - comp = ds4_cast_if_needed(ctx, comp, GGML_TYPE_F32); - kv_attn = ggml_concat(ctx, kv_attn, comp, 1); - } - if (old_rows_scratch) { - kv_attn = ggml_concat(ctx, kv_attn, old_rows_scratch, 1); + const bool fused_explicit_f16_kv = w.fused_verify_f16_kv && + masked_kv && n_tokens > 1 && + attention_impl == DeepSeek4AttentionImpl::Explicit && + kv_attn->type == GGML_TYPE_F32 && + raw_kv_source->type == GGML_TYPE_F16 && + (!comp_kv_source || comp_kv_source->type == GGML_TYPE_F16) && + (!old_rows_scratch_f16 || + old_rows_scratch_f16->type == GGML_TYPE_F16); + if (fused_explicit_f16_kv) { + // DS4's persistent MLA caches are already F16. Feed those tensors + // directly to the established explicit attention matmuls instead of + // casting the entire long-context cache to F32 on every verifier step. + // Current writes are consumed through their set_rows results, while + // preserved overwritten rows retain the same cached F16 values. + kv_attn = ggml_view_2d( + ctx, raw_kv_source, head_dim, n_raw, raw_kv_source->nb[1], 0); + if (n_comp_attn > 0 && comp_kv_source) { + ggml_tensor * comp = ggml_view_2d( + ctx, comp_kv_source, head_dim, n_comp_attn, + comp_kv_source->nb[1], 0); + kv_attn = ggml_concat(ctx, kv_attn, comp, 1); + } + if (old_rows_scratch_f16) { + kv_attn = ggml_concat( + ctx, kv_attn, old_rows_scratch_f16, 1); + } + static std::atomic explicit_f16_kv_logged{false}; + if (!explicit_f16_kv_logged.exchange(true)) { + std::fprintf(stderr, + "[deepseek4] fused explicit F16 K/V active: tokens=%d " + "compressed=%d\n", + n_tokens, n_comp_attn); + explicit_f16_kv_logged = true; + } + } else { + if (n_comp_attn > 0 && comp_kv_source) { + ggml_tensor * comp = ggml_view_2d( + ctx, comp_kv_source, head_dim, n_comp_attn, + comp_kv_source->nb[1], 0); + comp = ds4_cast_if_needed(ctx, comp, GGML_TYPE_F32); + kv_attn = ggml_concat(ctx, kv_attn, comp, 1); + } + if (old_rows_scratch) { + kv_attn = ggml_concat(ctx, kv_attn, old_rows_scratch, 1); + } } // kv_attn: [head_dim, n_attn] @@ -2235,6 +2280,22 @@ static ggml_tensor * build_mla_attention( ggml_tensor * q_flat = ggml_reshape_2d(ctx, q, head_dim, n_head * n_tokens); ggml_tensor * scores = ggml_mul_mat(ctx, kv_attn, q_flat); + const bool explicit_f16_f32_kv_short = fused_explicit_f16_kv && + n_attn <= DS4_FUSED_VERIFY_F16_F32_KV_MAX_ATTN; + if (explicit_f16_f32_kv_short) { + // Keep Q and accumulation in F32 while retaining the persistent + // cache in F16. The value-side matmul below uses the same bounded + // precision policy without changing cache topology. + ggml_mul_mat_set_prec(scores, GGML_PREC_F32); + static std::atomic f32_kv_short_logged{false}; + if (!f32_kv_short_logged.exchange(true)) { + std::fprintf(stderr, + "[deepseek4] fused explicit short-cache F32 K/V active: " + "n_attn=%d threshold=%d\n", n_attn, + DS4_FUSED_VERIFY_F16_F32_KV_MAX_ATTN); + f32_kv_short_logged = true; + } + } scores = ggml_scale(ctx, scores, kq_scale); if (score_mask) { if (n_tokens > 1) { @@ -2271,6 +2332,12 @@ static ggml_tensor * build_mla_attention( } ggml_tensor * kv_t = ggml_cont(ctx, ggml_transpose(ctx, kv_attn)); context = ggml_mul_mat(ctx, kv_t, probs); + if (explicit_f16_f32_kv_short) { + // Match the pre-existing F32-cache path on the value-side matmul + // as well. The same short-window bound contains its conversion + // cost, while probabilities and accumulation stay in F32. + ggml_mul_mat_set_prec(context, GGML_PREC_F32); + } context = ggml_reshape_3d(ctx, context, head_dim, n_head, n_tokens); } diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index eea667423..08d269c61 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -239,6 +239,7 @@ struct DeepSeek4Weights { // GGUF is loaded; they are not model metadata. int routed_expert_top_k = 0; // 0 = model default (n_expert_used) bool fused_decode = false; + bool fused_verify_f16_kv = false; }; inline bool deepseek4_is_eos_tok(int tok, const DeepSeek4Weights & w) { @@ -313,6 +314,7 @@ struct DeepSeek4BackendConfig { int max_ctx = 0; // 0 = auto from SWA + compression capacity int expert_top_k = 0; // 0 = use all model-routed experts bool fused_decode = false; // single-graph GPU decode + bool fused_verify_f16_kv = false; // F16 KV in batched verifier attention }; // ─── Function declarations ────────────────────────────────────────────── diff --git a/server/src/server/server_main.cpp b/server/src/server/server_main.cpp index 7373aace4..d83e43d50 100644 --- a/server/src/server/server_main.cpp +++ b/server/src/server/server_main.cpp @@ -97,6 +97,8 @@ static void print_usage(const char * prog) { " --peer-access Enable peer access for multi-GPU placement\n" " --chunk Chunked-prefill chunk size (default: 512)\n" " --ds4-fused-decode Enable DeepSeek4 single-graph GPU decode\n" + " --ds4-fused-verify-f16-kv\n" + " Reuse F16 MLA cache in batched DeepSeek4 verification\n" " --ds4-expert-top-k \n" " Keep and renormalize the highest-ranked N routed experts\n" " (0=model default; single-device DeepSeek4 only)\n" @@ -338,6 +340,8 @@ int main(int argc, char ** argv) { bargs.chunk = std::atoi(argv[++i]); } else if (std::strcmp(argv[i], "--ds4-fused-decode") == 0) { bargs.ds4_fused_decode = true; + } else if (std::strcmp(argv[i], "--ds4-fused-verify-f16-kv") == 0) { + bargs.ds4_fused_verify_f16_kv = true; } else if (std::strcmp(argv[i], "--ds4-expert-top-k") == 0 && i + 1 < argc) { bargs.ds4_expert_top_k = std::atoi(argv[++i]); if (bargs.ds4_expert_top_k < 0) { @@ -1092,6 +1096,8 @@ int main(int argc, char ** argv) { if (arch == "deepseek4") { std::fprintf(stderr, "[server] │ ds4_fused = %s\n", bargs.ds4_fused_decode ? "ON" : "off"); + std::fprintf(stderr, "[server] │ ds4_verify_f16kv= %s\n", + bargs.ds4_fused_verify_f16_kv ? "ON" : "off"); if (bargs.ds4_expert_top_k > 0) { std::fprintf(stderr, "[server] │ ds4_expert_topk= %d\n", bargs.ds4_expert_top_k); diff --git a/server/test/test_feature_gate.cpp b/server/test/test_feature_gate.cpp index d01b9da04..001755d36 100644 --- a/server/test/test_feature_gate.cpp +++ b/server/test/test_feature_gate.cpp @@ -271,6 +271,18 @@ void test_feature_gate_ds4_decode_options_require_monolithic_hip() { CHECK(gate_result( fused, "deepseek4", PlacementBackend::Hip).empty()); + BackendArgs f16_kv = gate_args_hip_deepseek4(); + f16_kv.ds4_fused_verify_f16_kv = true; + CHECK(!gate_result( + f16_kv, "deepseek4", PlacementBackend::Cuda).empty()); + CHECK(gate_result( + f16_kv, "deepseek4", PlacementBackend::Hip).empty()); + + BackendArgs split_f16_kv = f16_kv; + split_f16_kv.device.layer_split_gpus = {0, 1}; + CHECK(!gate_result( + split_f16_kv, "deepseek4", PlacementBackend::Hip).empty()); + BackendArgs topk = gate_args_hip_deepseek4(); topk.ds4_expert_top_k = 4; CHECK(!gate_result(