perf(ds4): reuse F16 KV in fused verification

server/docs/DS4.md

@@ -343,9 +343,18 @@ export DFLASH_DS4_SPEC_Q=4
 
 ./server/build-hip/dflash_server /path/to/deepseek4-target.gguf \
   --target-device hip:0 \
+  --ds4-fused-verify-f16-kv \
   --ds4-fused-decode
 ```
 
+`--ds4-fused-verify-f16-kv` feeds the persistent F16 MLA cache directly to
+batched explicit verifier attention instead of converting the full cache to
+F32 on every speculative step. Key-side accumulation remains F32 through 512
+attention rows to preserve the short-context quality baseline. The option is
+currently qualified only for a single HIP target and remains off by default.
+It changes verifier floating-point inputs and can change generated tokens, so
+re-run workload quality checks before enabling it for another checkpoint.
+
 `DFLASH_DS4_FUSED_VERIFY=1` is the opt-in throughput profile. Its persistent
 whole-model GPU graph uses stable padded reduction shapes, so near-tied greedy
 logits can select a different token than the normal causal verifier even at

server/src/common/backend_args.h

@@ -59,6 +59,7 @@ struct BackendArgs {
     // deepseek4-specific decode options
     int             ds4_expert_top_k = 0;  // 0 = model default
     bool            ds4_fused_decode = false;
+    bool            ds4_fused_verify_f16_kv = false;
 
     // Attention and speculative-decode options. Individual backends consume
     // only the fields they support.

server/src/common/backend_factory.cpp

@@ -424,6 +424,7 @@ std::unique_ptr<ModelBackend> create_backend(
             cfg.chunk      = args.chunk;
             cfg.expert_top_k = args.ds4_expert_top_k;
             cfg.fused_decode = args.ds4_fused_decode;
+            cfg.fused_verify_f16_kv = args.ds4_fused_verify_f16_kv;
             cfg.prefill_mode = args.ds4_prefill_mode;
 
             auto backend = std::make_unique<DeepSeek4Backend>(cfg);

server/src/common/feature_gate.cpp

@@ -280,6 +280,12 @@ std::string check_feature_compatibility(
                "DeepSeek4";
     }
 
+    // ── --ds4-fused-verify-f16-kv × placement
+    if (args.ds4_fused_verify_f16_kv && !monolithic_ds4) {
+        return "--ds4-fused-verify-f16-kv currently requires single-device "
+               "HIP DeepSeek4";
+    }
+
     // ── --ds4-expert-top-k × architecture/adapter
     if (args.ds4_expert_top_k != 0 && !local_ds4) {
         return "--ds4-expert-top-k currently requires a single local "

server/src/deepseek4/deepseek4_backend.cpp

@@ -674,7 +674,7 @@ DeepSeek4Backend::~DeepSeek4Backend() {
 }
 
 bool DeepSeek4Backend::requires_monolithic_model() const {
-    return cfg_.fused_decode ||
+    return cfg_.fused_decode || cfg_.fused_verify_f16_kv ||
            prefill_attention_mode_is_approximate(cfg_.prefill_mode);
 }
 
@@ -719,9 +719,11 @@ bool DeepSeek4Backend::load_model() {
         (force_full || need_monolithic)) {
         std::fprintf(stderr,
                      "[deepseek4] monolithic execution requested "
-                     "(forced=%s, fused_decode=%s, prefill=%s)\n",
+                     "(forced=%s, fused_decode=%s, "
+                     "fused_verify_f16_kv=%s, prefill=%s)\n",
                      force_full ? "yes" : "no",
                      cfg_.fused_decode ? "on" : "off",
+                     cfg_.fused_verify_f16_kv ? "on" : "off",
                      prefill_attention_mode_name(cfg_.prefill_mode));
         if (!load_deepseek4_gguf(cfg_.model_path, backend_, w_)) {
             if (prefill_attention_mode_is_approximate(cfg_.prefill_mode)) {
@@ -758,11 +760,17 @@ bool DeepSeek4Backend::load_model() {
     }
     w_.routed_expert_top_k = cfg_.expert_top_k;
     w_.fused_decode = cfg_.fused_decode && !moe_hybrid_;
+    w_.fused_verify_f16_kv = cfg_.fused_verify_f16_kv && !moe_hybrid_;
     if (cfg_.fused_decode && moe_hybrid_) {
         std::fprintf(stderr,
                      "[deepseek4] fused decode unavailable with hybrid expert placement; "
                      "using layered decode\n");
     }
+    if (cfg_.fused_verify_f16_kv && moe_hybrid_) {
+        std::fprintf(stderr,
+                     "[deepseek4] fused verifier F16 K/V unavailable with hybrid "
+                     "expert placement; using F32 verifier attention\n");
+    }
     return true;
 }
 

server/src/deepseek4/deepseek4_graph.cpp

@@ -60,6 +60,11 @@ static bool ds4_env_flag(const char * name) {
     return value && value[0] && std::strcmp(value, "0") != 0;
 }
 
+// F32 key/value-side accumulation protects the short-context quality baseline
+// while still avoiding a full-cache F16 -> F32 conversion once attention is
+// large enough for that conversion to dominate verifier time.
+static constexpr int DS4_FUSED_VERIFY_F16_F32_KV_MAX_ATTN = 512;
+
 static int ds4_effective_expert_count(const DeepSeek4Weights & w) {
     int requested = w.routed_expert_top_k;
     if (const char * value = std::getenv("DFLASH_DS4_TOPK")) {
@@ -1701,6 +1706,7 @@ static ggml_tensor * build_mla_attention(
     const bool layer_major_batch =
         causal_batch && attention_impl != DeepSeek4AttentionImpl::Explicit;
     ggml_tensor * old_rows_scratch = nullptr;
+    ggml_tensor * old_rows_scratch_f16 = nullptr;
     int n_old_rows = 0;
     ggml_tensor * prior_rows_scratch = nullptr;
     int n_prior_rows = 0;
@@ -1718,6 +1724,7 @@ static ggml_tensor * build_mla_attention(
                 ? ggml_concat(ctx, old_rows_scratch, saved, 1) : saved;
             n_old_rows++;
         }
+        old_rows_scratch_f16 = old_rows_scratch;
         old_rows_scratch = ds4_cast_if_needed(ctx, old_rows_scratch, GGML_TYPE_F32);
     } else if (causal_batch && !layer_major_batch) {
         // Copy the to-be-overwritten rows FIRST; same-stream build order runs
@@ -1947,13 +1954,51 @@ static ggml_tensor * build_mla_attention(
     } else {
         kv_attn = raw_kv_view(0, n_raw);
     }
-    if (n_comp_attn > 0 && comp_kv_source) {
-        ggml_tensor * comp = ggml_view_2d(ctx, comp_kv_source, head_dim, n_comp_attn, comp_kv_source->nb[1], 0);
-        comp = ds4_cast_if_needed(ctx, comp, GGML_TYPE_F32);
-        kv_attn = ggml_concat(ctx, kv_attn, comp, 1);
-    }
-    if (old_rows_scratch) {
-        kv_attn = ggml_concat(ctx, kv_attn, old_rows_scratch, 1);
+    const bool fused_explicit_f16_kv = w.fused_verify_f16_kv &&
+        masked_kv && n_tokens > 1 &&
+        attention_impl == DeepSeek4AttentionImpl::Explicit &&
+        kv_attn->type == GGML_TYPE_F32 &&
+        raw_kv_source->type == GGML_TYPE_F16 &&
+        (!comp_kv_source || comp_kv_source->type == GGML_TYPE_F16) &&
+        (!old_rows_scratch_f16 ||
+         old_rows_scratch_f16->type == GGML_TYPE_F16);
+    if (fused_explicit_f16_kv) {
+        // DS4's persistent MLA caches are already F16. Feed those tensors
+        // directly to the established explicit attention matmuls instead of
+        // casting the entire long-context cache to F32 on every verifier step.
+        // Current writes are consumed through their set_rows results, while
+        // preserved overwritten rows retain the same cached F16 values.
+        kv_attn = ggml_view_2d(
+            ctx, raw_kv_source, head_dim, n_raw, raw_kv_source->nb[1], 0);
+        if (n_comp_attn > 0 && comp_kv_source) {
+            ggml_tensor * comp = ggml_view_2d(
+                ctx, comp_kv_source, head_dim, n_comp_attn,
+                comp_kv_source->nb[1], 0);
+            kv_attn = ggml_concat(ctx, kv_attn, comp, 1);
+        }
+        if (old_rows_scratch_f16) {
+            kv_attn = ggml_concat(
+                ctx, kv_attn, old_rows_scratch_f16, 1);
+        }
+        static std::atomic<bool> explicit_f16_kv_logged{false};
+        if (!explicit_f16_kv_logged.exchange(true)) {
+            std::fprintf(stderr,
+                "[deepseek4] fused explicit F16 K/V active: tokens=%d "
+                "compressed=%d\n",
+                n_tokens, n_comp_attn);
+            explicit_f16_kv_logged = true;
+        }
+    } else {
+        if (n_comp_attn > 0 && comp_kv_source) {
+            ggml_tensor * comp = ggml_view_2d(
+                ctx, comp_kv_source, head_dim, n_comp_attn,
+                comp_kv_source->nb[1], 0);
+            comp = ds4_cast_if_needed(ctx, comp, GGML_TYPE_F32);
+            kv_attn = ggml_concat(ctx, kv_attn, comp, 1);
+        }
+        if (old_rows_scratch) {
+            kv_attn = ggml_concat(ctx, kv_attn, old_rows_scratch, 1);
+        }
     }
     // kv_attn: [head_dim, n_attn]
 
@@ -2235,6 +2280,22 @@ static ggml_tensor * build_mla_attention(
         ggml_tensor * q_flat = ggml_reshape_2d(ctx, q, head_dim,
                                                n_head * n_tokens);
         ggml_tensor * scores = ggml_mul_mat(ctx, kv_attn, q_flat);
+        const bool explicit_f16_f32_kv_short = fused_explicit_f16_kv &&
+            n_attn <= DS4_FUSED_VERIFY_F16_F32_KV_MAX_ATTN;
+        if (explicit_f16_f32_kv_short) {
+            // Keep Q and accumulation in F32 while retaining the persistent
+            // cache in F16. The value-side matmul below uses the same bounded
+            // precision policy without changing cache topology.
+            ggml_mul_mat_set_prec(scores, GGML_PREC_F32);
+            static std::atomic<bool> f32_kv_short_logged{false};
+            if (!f32_kv_short_logged.exchange(true)) {
+                std::fprintf(stderr,
+                    "[deepseek4] fused explicit short-cache F32 K/V active: "
+                    "n_attn=%d threshold=%d\n", n_attn,
+                    DS4_FUSED_VERIFY_F16_F32_KV_MAX_ATTN);
+                f32_kv_short_logged = true;
+            }
+        }
         scores = ggml_scale(ctx, scores, kq_scale);
         if (score_mask) {
             if (n_tokens > 1) {
@@ -2271,6 +2332,12 @@ static ggml_tensor * build_mla_attention(
         }
         ggml_tensor * kv_t = ggml_cont(ctx, ggml_transpose(ctx, kv_attn));
         context = ggml_mul_mat(ctx, kv_t, probs);
+        if (explicit_f16_f32_kv_short) {
+            // Match the pre-existing F32-cache path on the value-side matmul
+            // as well. The same short-window bound contains its conversion
+            // cost, while probabilities and accumulation stay in F32.
+            ggml_mul_mat_set_prec(context, GGML_PREC_F32);
+        }
         context = ggml_reshape_3d(ctx, context, head_dim, n_head, n_tokens);
     }
 

server/src/deepseek4/deepseek4_internal.h

@@ -239,6 +239,7 @@ struct DeepSeek4Weights {
     // GGUF is loaded; they are not model metadata.
     int  routed_expert_top_k = 0;  // 0 = model default (n_expert_used)
     bool fused_decode        = false;
+    bool fused_verify_f16_kv = false;
 };
 
 inline bool deepseek4_is_eos_tok(int tok, const DeepSeek4Weights & w) {
@@ -313,6 +314,7 @@ struct DeepSeek4BackendConfig {
     int          max_ctx      = 0;     // 0 = auto from SWA + compression capacity
     int          expert_top_k = 0;     // 0 = use all model-routed experts
     bool         fused_decode = false; // single-graph GPU decode
+    bool         fused_verify_f16_kv = false; // F16 KV in batched verifier attention
 };
 
 // ─── Function declarations ──────────────────────────────────────────────

server/src/server/server_main.cpp

@@ -97,6 +97,8 @@ static void print_usage(const char * prog) {
         "  --peer-access        Enable peer access for multi-GPU placement\n"
         "  --chunk <N>          Chunked-prefill chunk size (default: 512)\n"
         "  --ds4-fused-decode   Enable DeepSeek4 single-graph GPU decode\n"
+        "  --ds4-fused-verify-f16-kv\n"
+        "                       Reuse F16 MLA cache in batched DeepSeek4 verification\n"
         "  --ds4-expert-top-k <N>\n"
         "                       Keep and renormalize the highest-ranked N routed experts\n"
         "                       (0=model default; single-device DeepSeek4 only)\n"
@@ -338,6 +340,8 @@ int main(int argc, char ** argv) {
             bargs.chunk = std::atoi(argv[++i]);
         } else if (std::strcmp(argv[i], "--ds4-fused-decode") == 0) {
             bargs.ds4_fused_decode = true;
+        } else if (std::strcmp(argv[i], "--ds4-fused-verify-f16-kv") == 0) {
+            bargs.ds4_fused_verify_f16_kv = true;
         } else if (std::strcmp(argv[i], "--ds4-expert-top-k") == 0 && i + 1 < argc) {
             bargs.ds4_expert_top_k = std::atoi(argv[++i]);
             if (bargs.ds4_expert_top_k < 0) {
@@ -1092,6 +1096,8 @@ int main(int argc, char ** argv) {
     if (arch == "deepseek4") {
         std::fprintf(stderr, "[server] │  ds4_fused      = %s\n",
                      bargs.ds4_fused_decode ? "ON" : "off");
+        std::fprintf(stderr, "[server] │  ds4_verify_f16kv= %s\n",
+                     bargs.ds4_fused_verify_f16_kv ? "ON" : "off");
         if (bargs.ds4_expert_top_k > 0) {
             std::fprintf(stderr, "[server] │  ds4_expert_topk= %d\n",
                          bargs.ds4_expert_top_k);

server/test/test_feature_gate.cpp

@@ -271,6 +271,18 @@ void test_feature_gate_ds4_decode_options_require_monolithic_hip() {
     CHECK(gate_result(
         fused, "deepseek4", PlacementBackend::Hip).empty());
 
+    BackendArgs f16_kv = gate_args_hip_deepseek4();
+    f16_kv.ds4_fused_verify_f16_kv = true;
+    CHECK(!gate_result(
+        f16_kv, "deepseek4", PlacementBackend::Cuda).empty());
+    CHECK(gate_result(
+        f16_kv, "deepseek4", PlacementBackend::Hip).empty());
+
+    BackendArgs split_f16_kv = f16_kv;
+    split_f16_kv.device.layer_split_gpus = {0, 1};
+    CHECK(!gate_result(
+        split_f16_kv, "deepseek4", PlacementBackend::Hip).empty());
+
     BackendArgs topk = gate_args_hip_deepseek4();
     topk.ds4_expert_top_k = 4;
     CHECK(!gate_result(