feat(kvcache): batch ContainsKeys and BeginLoads APIs

.github/workflows/ci.yml

@@ -40,6 +40,16 @@ jobs:
 
       # ── C++ core engine ──────────────────────────────────────────
 
+      - name: Install required OS packages
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends \
+            sudo curl ca-certificates gdb ccache rsync git \
+            build-essential cmake pkg-config \
+            libcurl4-openssl-dev libssl-dev libgflags-dev libzstd-dev \
+            libboost-context-dev libc-ares-dev libprotobuf-dev libprotoc-dev protobuf-compiler \
+            libjsoncpp-dev libleveldb-dev libsnappy-dev libzmq3-dev cppzmq-dev zlib1g-dev lcov
+
       - name: Configure CMake
         run: |
           cmake -B build -S . \
@@ -146,7 +156,7 @@ jobs:
         run: ccache --show-stats || true
 
       - name: Remove ci-approved label after run
-        if: ${{ always() }}
+        if: ${{ always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
         uses: actions-ecosystem/action-remove-labels@v1
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -94,7 +94,7 @@ bash scripts/install_dependency_ubuntu2404.sh
 
 This script installs all necessary dependencies including:
 - Build tools (CMake, GCC, Ninja)
-- System libraries (Boost, glog, jsoncpp, liburing, zstd, etc.)
+- System libraries (Boost, glog, jsoncpp, liburing, ZeroMQ/cppzmq, zstd, etc.)
 - AWS SDK C++ (S3)
 - Testing framework (Catch2)
 - Additional libraries (Abseil, gRPC, etc.)

docs/vllm_kvcache_guide.md

@@ -198,67 +198,191 @@ Check the server:
 
 ## Benchmark
 
-The built-in benchmark used during validation is:
+The built-in benchmark is
+`vllm/benchmarks/multi_turn/benchmark_serving_multi_turn.py`.
 
-```text
-vllm/benchmarks/multi_turn/benchmark_serving_multi_turn.py
+EloqStore ships a small generator for repeatable synthetic conversation files.
+Generate the inputs locally instead of committing generated JSON into the
+repository:
+
+```bash
+CONFIGS=/tmp/eloqstore-kvcache-bench-configs
+eloqstore-generate-kvcache-bench --output-dir "$CONFIGS"
+```
+
+| File | Conversations | First-turn tokens | Total KV cache | vs 5 GB |
+|------|--------------|-------------------|----------------|---------|
+| `fit_5g_conversations.json` | 3 | ~9000 | ~3.4 GB | Within budget |
+| `overflow_5g_conversations.json` | 7 | ~9000 | ~7.9 GB | Exceeds budget |
+| `high_hit_conversations.json` | 64 | ~1000 | ~8.95 GB | Exceeds budget |
+
+Each conversation: long first-turn analysis, short assistant reply, short
+second-turn follow-up question.
+
+Per-token KV cache size (Qwen3-4B, half):
+```
+2 × 36 layers × 8 kv_heads × 128 dim × 2 bytes = 147,456 bytes ≈ 144 KB
+```
+
+### Prerequisites
+
+```bash
+uv pip install -r /path/to/vllm/benchmarks/multi_turn/requirements.txt
 ```
 
-Example command:
+### Running a Benchmark
+
+Start the server (EloqStore or CPU offloading, see [Startup](#startup) above).
 
 ```bash
-/path/to/venv/bin/python \
-  /path/to/vllm/benchmarks/multi_turn/benchmark_serving_multi_turn.py \
+# Cold run (empty store / empty CPU buffer)
+VENV=/path/to/venv/bin
+VLLM_BENCH=/path/to/vllm/benchmarks/multi_turn/benchmark_serving_multi_turn.py
+CONFIGS=/tmp/eloqstore-kvcache-bench-configs
+eloqstore-generate-kvcache-bench --preset overflow-5g --output-dir "$CONFIGS"
+
+$VENV/python $VLLM_BENCH \
+  --model Qwen/Qwen3-4B \
+  --served-model-name qwen3-4b-eloq \
+  --url http://127.0.0.1:8015 \
+  --input-file "$CONFIGS/overflow_5g_conversations.json" \
+  --num-clients 1 \
+  --max-active-conversations 7 \
+  --max-turns 4 \
+  --no-early-stop \
+  --request-timeout-sec 300 \
+  --stats-json-output /path/to/eloq_overflow_cold.json
+
+# Warm run (data now in store / CPU buffer)
+$VENV/python $VLLM_BENCH \
   --model Qwen/Qwen3-4B \
   --served-model-name qwen3-4b-eloq \
   --url http://127.0.0.1:8015 \
-  --input-file /path/to/high_hit_over5g_conversations.json \
+  --input-file "$CONFIGS/overflow_5g_conversations.json" \
   --num-clients 1 \
-  --max-active-conversations 64 \
-  --max-num-requests 128 \
+  --max-active-conversations 7 \
   --max-turns 4 \
+  --no-early-stop \
   --request-timeout-sec 300 \
-  --stats-json-output /path/to/high_hit_stats_eloq.json
+  --stats-json-output /path/to/eloq_overflow_warm.json
 ```
 
-The benchmark input can either be:
+`--max-active-conversations` must equal the number of conversations in the
+input file.  `--no-early-stop` forces all turns to complete.
 
-- a synthetic generation config (`filetype: generate_conversations`)
-- or a literal list of OpenAI-format conversations
+Repeat with the CPU offloading connector
+(`--kv-transfer-config '{"kv_connector":"OffloadingConnector",...}'`) on a
+different port for comparison.
 
-For cache-hit testing, the literal conversation list is more useful because it
-lets you guarantee repeated multi-turn reuse.
+### Generating Reports
 
-## Current Measured Results
+The `eloqstore.bench_report` module (also available as the
+`eloqstore-bench-report` CLI) reads the `--stats-json-output` files and prints
+comparison tables.
 
-Using an explicit high-hit workload with:
+```bash
+# Full 4-way comparison (cold + warm for both systems)
+eloqstore-bench-report \
+  --eloq-cold   eloq_cold.json \
+  --eloq-warm   eloq_warm.json \
+  --offload-cold offload_cold.json \
+  --offload-warm offload_warm.json
+```
 
-- `64` conversations
-- total first-turn hotset above `5 GiB`
-- second-turn short questions that strongly reuse the first-turn history
+Or from Python:
 
-Current best validated EloqStore result:
+```python
+from eloqstore.bench_report import load_stats, print_report
 
-```text
-requests_per_sec = 6.879
-ttft_ms mean     = 128.95
-latency_ms mean  = 143.21
+print_report(
+    eloq_cold=load_stats("eloq_cold.json"),
+    eloq_warm=load_stats("eloq_warm.json"),
+    offload_cold=load_stats("offload_cold.json"),
+    offload_warm=load_stats("offload_warm.json"),
+)
 ```
 
-Reference CPU memory offloading result on the same workload:
+## Current Measured Results
+
+Test setup:
 
-```text
-requests_per_sec = 8.665
-ttft_ms mean     = 92.78
-latency_ms mean  = 113.26
+- Model: Qwen3-4B, dtype half, enforce eager
+- GPU: RTX 5080 (16 GB), gpu-memory-utilization 0.60
+- KV cache budget: 5 GB (EloqStore shared buffer / CPU RAM)
+- vLLM base: v0.22.0 with EloqStore batch API
+- Per-token KV cache: ~144 KB (36 layers × 8 kv_heads × 128 dim × 2 bytes)
+
+### Workload A — Fit (KV cache fits within 5 GB)
+
+3 conversations, each ~9,000-token first turn (≈ 6,500 measured input tokens).
+
+| System | Cold TTFT | Warm TTFT | Speedup |
+|--------|-----------|-----------|---------|
+| CPU Offloading | 505ms | 62ms | **8.10×** |
+| EloqStore | 937ms | 222ms | **4.22×** |
+
+All blocks fit in the CPU ring buffer.  CPU offloading loads from RAM
+(~100ns latency).  EloqStore loads from NVMe via io_uring (~10µs latency).
+Both show significant speedup; CPU is faster due to storage medium.
+
+### Workload B — Overflow (KV cache exceeds 5 GB)
+
+7 conversations, each ~9,000-token first turn (≈ 6,500 measured input tokens).
+
+| System | Cold TTFT | Warm TTFT | Speedup |
+|--------|-----------|-----------|---------|
+| CPU Offloading | 634ms | 568ms | **1.12×** |
+| EloqStore | 1013ms | 331ms | **3.06×** |
+
+The hotset exceeds the 5 GB CPU buffer.  CPU ring buffer evicts ~2.9 GB
+(37%) of blocks.  Evicted blocks must be recomputed on GPU — negates nearly
+all cache benefit.  EloqStore persists to NVMe SSD — no eviction — 3.06×
+speedup.
+
+### Workload C — High Cache Hit (small requests, large hotset)
+
+64 conversations, ~1,000-token first turns (≈ 1,000 measured input tokens),
+second-turn short questions.  Total KV cache ~8.95 GB >> 5 GB.
+
+| System | Cold TTFT | Warm TTFT | Speedup |
+|--------|-----------|-----------|---------|
+| CPU Offloading | 92ms | 91ms | **1.02×** |
+| EloqStore | 166ms | 75ms | **2.22×** |
+
+On this workload, EloqStore warm-cache **outperforms** CPU offloading
+(75ms vs 91ms).  The batch `ContainsKeys` and `BeginLoads` APIs eliminate
+per-key synchronization overhead on the warm path.
+
+### Summary
+
+TTFT absolute values differ across workloads because GPU prefill time scales
+with input tokens (~9,000 tokens ≈ 600–900ms prefill; ~1,000 tokens ≈
+100–150ms).  The speedup column isolates the cache benefit independent of
+prompt length.
+
+```
+                    FIT (9K tok, 3.4 GB)  OVERFLOW (9K tok, 7.9 GB)  HIGH HIT (1K tok, 8.95 GB)
+                    cold    warm  speedup  cold    warm  speedup      cold    warm  speedup
+CPU Offloading      505ms   62ms  8.10x   634ms  568ms  1.12x        92ms    91ms  1.02x
+EloqStore           937ms  222ms  4.22x  1013ms  331ms  3.06x       166ms    75ms  2.22x
 ```
 
-So the current EloqStore implementation is still slower than CPU offloading,
-but the gap has been reduced substantially by:
+1. **When KV cache fits in RAM**: CPU offloading wins (8.10× vs 4.22×).
+   RAM is ~100× faster than NVMe SSD for random reads.
+
+2. **When KV cache exceeds RAM**: CPU offloading loses nearly all benefit
+   (1.12×) due to ring-buffer eviction.  EloqStore maintains substantial
+   speedup (3.06×) because SSDs have orders of magnitude more capacity.
+
+3. **EloqStore can outperform CPU offloading**: On the high-hit workload,
+   EloqStore warm-cache (75ms) beats CPU offloading (91ms).  Batch APIs and
+   persistent storage together make EloqStore competitive even against
+   RAM-backed caches when the working set is large.
 
-- block-mapped shared memory
-- memory-only prefix matching
-- lighter save/load data path handling
+4. **Capacity is the differentiator**: CPU offloading provides fast access
+   for hot caches that fit within available RAM.  EloqStore provides
+   predictable cache reuse regardless of working-set size, bounded only by
+   available NVMe storage.
 
 ## Failure Modes
 

ffi/include/eloqstore_capi.h

@@ -358,6 +358,18 @@ extern "C"
     bool CEloqStore_KVCacheManager_ContainsKey(CKVCacheManagerHandle runtime,
                                                const char *key,
                                                bool *out_exists);
+    // Probe whether multiple keys exist through the manager runtime.
+    bool CEloqStore_KVCacheManager_ContainsKeys(CKVCacheManagerHandle runtime,
+                                                size_t num_keys,
+                                                const char *const *keys,
+                                                bool *out_exists);
+    // Begin batched asynchronous loads and return their request ids.
+    bool CEloqStore_KVCacheManager_BeginLoads(
+        CKVCacheManagerHandle runtime,
+        size_t num_keys,
+        const char *const *keys,
+        const uint32_t *payload_bytes_list,
+        uint64_t *out_request_ids);
 
     // Create one worker-side KV cache control-plane stub from native options.
     CKVCacheWorkerHandle CEloqStore_KVCacheWorker_Create(

ffi/src/eloqstore_capi.cpp

@@ -813,6 +813,84 @@ extern "C"
         return ok;
     }
 
+    bool CEloqStore_KVCacheManager_ContainsKeys(CKVCacheManagerHandle runtime,
+                                                size_t num_keys,
+                                                const char *const *keys,
+                                                bool *out_exists)
+    {
+        clear_last_error();
+        if (runtime == nullptr || keys == nullptr || out_exists == nullptr)
+        {
+            set_last_error("invalid manager contains-keys arguments");
+            return false;
+        }
+        std::vector<std::string> cpp_keys(num_keys);
+        for (size_t i = 0; i < num_keys; ++i)
+        {
+            if (keys[i] == nullptr)
+            {
+                set_last_error("contains-keys key is null");
+                return false;
+            }
+            cpp_keys[i] = keys[i];
+        }
+        std::vector<bool> cpp_exists;
+        std::string error_message;
+        const bool ok =
+            reinterpret_cast<KVCacheManager *>(runtime)->ContainsKeys(
+                cpp_keys, &cpp_exists, &error_message);
+        if (!ok)
+        {
+            set_last_error(error_message);
+        }
+        for (size_t i = 0; i < num_keys; ++i)
+        {
+            out_exists[i] = cpp_exists[i];
+        }
+        return ok;
+    }
+
+    bool CEloqStore_KVCacheManager_BeginLoads(
+        CKVCacheManagerHandle runtime,
+        size_t num_keys,
+        const char *const *keys,
+        const uint32_t *payload_bytes_list,
+        uint64_t *out_request_ids)
+    {
+        clear_last_error();
+        if (runtime == nullptr || keys == nullptr ||
+            payload_bytes_list == nullptr || out_request_ids == nullptr)
+        {
+            set_last_error("invalid manager begin-loads arguments");
+            return false;
+        }
+        std::vector<std::string> cpp_keys(num_keys);
+        std::vector<uint32_t> cpp_payloads(num_keys);
+        for (size_t i = 0; i < num_keys; ++i)
+        {
+            if (keys[i] == nullptr)
+            {
+                set_last_error("begin-loads key is null");
+                return false;
+            }
+            cpp_keys[i] = keys[i];
+            cpp_payloads[i] = payload_bytes_list[i];
+        }
+        std::vector<uint64_t> cpp_req_ids;
+        std::string error_message;
+        const bool ok = reinterpret_cast<KVCacheManager *>(runtime)->BeginLoads(
+            cpp_keys, cpp_payloads, &cpp_req_ids, &error_message);
+        if (!ok)
+        {
+            set_last_error(error_message);
+        }
+        for (size_t i = 0; i < cpp_req_ids.size(); ++i)
+        {
+            out_request_ids[i] = cpp_req_ids[i];
+        }
+        return ok;
+    }
+
     CKVCacheWorkerHandle CEloqStore_KVCacheWorker_Create(
         CKVCacheOptionsHandle opts)
     {

include/sdk_runtime.h

@@ -140,11 +140,17 @@ class KVCacheManager
     bool ContainsKey(const std::string &key,
                      bool *out_exists,
                      std::string *error_message);
-    // Check whether one key already exists, first in memory then in EloqStore.
     bool ContainsKey(const std::string &key,
                      uint32_t partition_id,
                      bool *out_exists,
                      std::string *error_message);
+    bool ContainsKeys(const std::vector<std::string> &keys,
+                      std::vector<bool> *out_exists,
+                      std::string *error_message);
+    bool BeginLoads(const std::vector<std::string> &keys,
+                    const std::vector<uint32_t> &payload_bytes_list,
+                    std::vector<uint64_t> *out_request_ids,
+                    std::string *error_message);
     bool GetMetrics(KVCacheRuntimeMetrics *out_metrics,
                     std::string *error_message);
     const std::vector<ShardLayout> &shards() const