From 917a8d8a018f3c867c879dcdafbf34884676db6b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 7 Jun 2026 05:34:30 +0000 Subject: [PATCH 01/30] Sync version to 1.1.1 [skip ci] --- python/pyproject.toml | 0 rust/eloqstore-sys/Cargo.toml | 0 rust/eloqstore/Cargo.toml | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 python/pyproject.toml mode change 100644 => 100755 rust/eloqstore-sys/Cargo.toml mode change 100644 => 100755 rust/eloqstore/Cargo.toml diff --git a/python/pyproject.toml b/python/pyproject.toml old mode 100644 new mode 100755 diff --git a/rust/eloqstore-sys/Cargo.toml b/rust/eloqstore-sys/Cargo.toml old mode 100644 new mode 100755 diff --git a/rust/eloqstore/Cargo.toml b/rust/eloqstore/Cargo.toml old mode 100644 new mode 100755 From d654fd5d68026f9c4b92c6019d2abdf8c6b84606 Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Fri, 3 Jul 2026 20:11:09 -0700 Subject: [PATCH 02/30] =?UTF-8?q?feat(io):=20per-shard=20IO=20QoS=20?= =?UTF-8?q?=E2=80=94=20in-flight=20page-IO=20budgets=20with=20FG/BG=20read?= =?UTF-8?q?=20isolation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Background tasks (batch writes, compaction, GC) degraded foreground read tail latency even after the CPU-side yield work (#455): a background task scheduled for one 20us slice could still burst 128 page reads plus writes into io_uring, and nothing distinguished them from foreground reads at the ring or device. Reads had no budget at all; writes only a per-task one. CPU priority was lost at the IO boundary. Design: docs/design/io_qos.md (+ implementation/testing record in docs/design/io_qos_impl_plan.md). Mechanisms (per shard, all in IouringMgr): - IoBudget: in-flight page-IO counters with independent read/write caps (max_inflight_read / max_inflight_write, 4KB-page units; merged append writes cost bytes/page_size). Acquired immediately before SQE prep — after every other blocking acquisition — and released per CQE in PollComplete via new user-data types (KvTaskPageRead/BaseReqPageRead) that distinguish budgeted page reads from metadata ops. Batched reads acquire per page, so a 128-page compaction batch cannot deadlock against a small budget. A request costlier than the cap is admitted alone once the budget drains (merged writes vs small caps). Metadata, manifest, and segment IO are exempt. - Background read sub-budget (bg_read_ratio, KvTask::IsBackground()): background page reads are additionally bounded so compaction/GC/batch- write read bursts cannot crowd foreground point reads out of the device queue. While background waiters queue, their unused entitlement is reserved from new foreground admissions — sustained foreground saturation cannot starve compaction (and vice versa); release wakes background first and forwards unused wake credits (WaitingZone::WakeN now returns the count woken). The ratio parameterization is deliberate: with the total cap sized to the device's bandwidth-delay product, relative tail inflation is device-independent (see Sizing contract in the design doc). - Retired: the per-task max_write_batch_pages drain-to-zero throttle (option deprecated, parsed with a warning); max_inflight_write is now the write QoS cap (default 32768 -> 512, calibrated: natural demand ceiling was 2048 = 8 concurrent 1MB merged writes; 512 binds marginally at equal-or-best throughput and tails). Defaults max_inflight_read = 32, bg_read_ratio = 25 from interference sweeps (bg cap lands on the measured background demand line; write-throughput knee only below it). - Observability: IoQosStats (in-flight/high-watermark/blocked/admitted per budget + bg slice, fdatasync count/latency), EloqStore::GetIoQosStats, TXSERVICE gauges. Benchmarks/tooling: benchmark/interference_bench (baseline vs 90/10 write/read mixed phase; rotating strided partial-overwrite storm so compaction continuously relocates pages — full overwrites generate no move reads; exact percentiles; per-shard QoS deltas) and scripts/io_calibration_sweep.sh (fio read-tail-vs-write-rate curve). On the dev box the sub-budget cut mixed-phase read p99 ~2.5x and p99.9 3-5x at identical write throughput, with flat p50. Tests: tests/io_qos.cpp (11 cases: accounting invariants, oversized batches through tiny caps, BG sub-budget bounds under compaction, concurrent merged-write admission, failed-write budget drain, shutdown while blocked on a budget). Full suite green; db_stress 120/120 with tiny caps in both write modes; ASAN (flag-injection mode) clean. Two OOM tests that implicitly relied on the retired per-task drain to bound write-promotion pins now set explicit write budgets. Also: test harness now enforces one started EloqStore per process (InitStore gained cleanup=false for warm-restart tests; cloud tests no longer construct stores directly) — fixes a flaky teardown SIGSEGV where a test-local store Stop nulled the global eloq_store pointer that the lingering singleton prewarm shutdown later dereferenced. Deployment notes: max_inflight_write semantics changed for configs that set it explicitly (was pool sizing only, now a queue-depth cap); max_write_batch_pages is ignored. Real-device calibration should re-derive max_inflight_read (QD sweep); bg_read_ratio is policy and transfers. The M3 background write rate limiter remains deferred pending real-device evidence. --- benchmark/CMakeLists.txt | 4 + benchmark/interference_bench.cpp | 534 ++++++++++++++++++ benchmark/opts_interference.ini | 30 + docs/architecture/02-runtime-and-lifecycle.md | 9 +- docs/architecture/04-execution-model.md | 12 + docs/architecture/07-io-stack.md | 25 + docs/architecture/08-data-lifecycle.md | 6 + docs/design/io_qos.md | 388 +++++++++++++ docs/design/io_qos_impl_plan.md | 337 +++++++++++ include/async_io_manager.h | 157 ++++- include/eloq_store.h | 10 + include/eloqstore_metrics.h | 6 + include/kv_options.h | 45 +- include/storage/shard.h | 11 +- include/tasks/task.h | 28 +- scripts/io_calibration_sweep.sh | 130 +++++ src/async_io_manager.cpp | 156 ++++- src/eloq_store.cpp | 10 + src/kv_options.cpp | 15 + src/storage/shard.cpp | 9 + src/tasks/task.cpp | 7 +- src/tasks/write_task.cpp | 19 +- tests/CMakeLists.txt | 1 + tests/batch_write.cpp | 2 - tests/cloud.cpp | 22 +- tests/common.cpp | 21 +- tests/common.h | 11 +- tests/data_page_cache.cpp | 13 +- tests/io_qos.cpp | 405 +++++++++++++ 29 files changed, 2377 insertions(+), 46 deletions(-) create mode 100644 benchmark/interference_bench.cpp create mode 100644 benchmark/opts_interference.ini create mode 100644 docs/design/io_qos.md create mode 100644 docs/design/io_qos_impl_plan.md create mode 100755 scripts/io_calibration_sweep.sh create mode 100644 tests/io_qos.cpp diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 6ad4d8e5a..6cf29b878 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -62,6 +62,10 @@ target_link_libraries(load_bench eloqstore ${BRPC_LIB} ${GFLAGS_LIBRARY}) add_executable(simple_bench simple_bench.cpp) target_link_libraries(simple_bench eloqstore ${BRPC_LIB} ${GFLAGS_LIBRARY}) +# IO QoS interference benchmark (docs/design/io_qos.md, plan commit 3). +add_executable(interference_bench interference_bench.cpp) +target_link_libraries(interference_bench eloqstore ${GFLAGS_LIBRARY}) + add_executable(simple_test simple_test.cpp) target_link_libraries(simple_test eloqstore ${BRPC_LIB} ${GFLAGS_LIBRARY}) diff --git a/benchmark/interference_bench.cpp b/benchmark/interference_bench.cpp new file mode 100644 index 000000000..3916b29c1 --- /dev/null +++ b/benchmark/interference_bench.cpp @@ -0,0 +1,534 @@ +/** + * IO QoS interference benchmark (docs/design/io_qos.md, plan commit 3). + * + * Measures how much a background write/compaction storm degrades foreground + * point-read tail latency, and how the IO QoS knobs (max_inflight_read, + * bg_read_ratio, max_inflight_write) change that. + * + * Phases: + * 1. load — fill P partitions with K keys of ~val_size bytes each. + * val_size defaults to 3000 so one KV fills one 4KB data + * page: key granularity == page granularity, which lets the + * storm control per-file liveness exactly. + * 2. baseline — closed-loop uniform-random point reads at fixed + * concurrency for baseline_secs. No writes. + * 3. mixed — the measured workload: a write-dominated op mix (default + * 90% write key-ops / 10% point reads, --write_read_ratio) + * where reads are paced off completed write batches so the + * ratio holds regardless of relative speeds; set + * write_read_ratio=0 for the original unthrottled + * reads-vs-storm shape. The writes overwrite a rotating + * strided subset of keys (span of every ratio, default 3 + * of 5, shifted by one each round), keeping every data + * file ~40% live — above file_amplify_factor — so + * compaction continuously relocates live pages through + * 128-page ReadPages bursts. (A full overwrite would leave + * files 100% dead: compaction just drops them and generates + * NO read traffic — see io_qos_impl_plan.md.) + * + * Reports per phase: read QPS and exact p50/p90/p99/p99.9/max latency + * (computed from raw samples, not a sliding window), the storm's write MB/s, + * and per-shard IoQosStats deltas (in-flight watermarks, budget-blocked + * counts/time, fdatasync). Greppable one-line summaries are prefixed with + * "RESULT" for sweep scripts. + * + * EloqStore options (including the QoS knobs) come from --kvoptions ini, so + * sweeps only vary the ini / flags. See opts_interference.ini. + */ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "async_io_manager.h" // IoQosStats +#include "coding.h" +#include "eloq_store.h" +#include "utils.h" + +// https://github.com/cameron314/concurrentqueue/issues/280 +#undef BLOCK_SIZE +#include "../external/concurrentqueue/blockingconcurrentqueue.h" + +DEFINE_string(kvoptions, "", "Path to EloqStore options ini"); +DEFINE_uint32(partitions, 4, "number of partitions"); +DEFINE_uint32(keys_per_partition, 20000, "keys per partition"); +DEFINE_uint32(val_size, + 3000, + "value bytes; default ~3000 = one KV per 4KB data page"); +DEFINE_uint32(read_concurrency, 32, "concurrent point reads (closed loop)"); +DEFINE_uint32(baseline_secs, 15, "seconds of read-only baseline phase"); +DEFINE_uint32(storm_secs, 60, "seconds of read + write-storm phase"); +DEFINE_uint32(storm_ratio, 5, "overwrite span out of every ratio keys"); +DEFINE_uint32(storm_span, 3, "overwrite span out of every ratio keys"); +DEFINE_uint32(storm_batch_keys, 2048, "keys per storm batch-write request"); +DEFINE_uint32(write_read_ratio, + 9, + "write key-ops per point read in the mixed phase (9 = 90/10 " + "write/read op mix). 0 = reads run unthrottled closed-loop " + "alongside the writes (the original storm shape)"); +DEFINE_bool(load, true, "load data first (false reuses an existing store)"); + +using namespace std::chrono; + +namespace +{ +constexpr char kTable[] = "ifb"; + +std::atomic g_phase{0}; // 0 load, 1 baseline, 2 mixed, 3 done + +// Read pacing for the mixed phase: every completed storm batch adds its key +// count; issuing one read consumes write_read_ratio credits, so reads track +// the write throughput at the configured op ratio (e.g. 9 -> 10% reads / +// 90% writes by ops). +std::atomic g_read_credits{0}; + +bool TryConsumeReadCredits() +{ + const int64_t need = FLAGS_write_read_ratio; + int64_t cur = g_read_credits.load(std::memory_order_relaxed); + while (cur >= need) + { + if (g_read_credits.compare_exchange_weak( + cur, cur - need, std::memory_order_relaxed)) + { + return true; + } + } + return false; +} + +void EncodeKey(char *dst, uint64_t key) +{ + eloqstore::EncodeFixed64(dst, eloqstore::ToBigEndian(key)); +} + +std::string MakeKey(uint64_t key) +{ + std::string s; + s.resize(sizeof(uint64_t)); + EncodeKey(s.data(), key); + return s; +} + +// ---------------------------------------------------------------- readers + +struct Reader +{ + explicit Reader(uint32_t id) : id_(id) + { + } + const uint32_t id_; + eloqstore::ReadRequest request_; + char key_[sizeof(uint64_t)]; + uint64_t start_us_{0}; + int issue_phase_{0}; +}; + +struct PhaseLatencies +{ + std::vector samples; // microseconds + uint64_t not_found{0}; + uint64_t errors{0}; +}; + +uint64_t Percentile(std::vector &sorted, double p) +{ + if (sorted.empty()) + { + return 0; + } + size_t idx = static_cast(p * (sorted.size() - 1)); + return sorted[idx]; +} + +void ReportPhase(const char *name, PhaseLatencies &lat, double secs) +{ + std::sort(lat.samples.begin(), lat.samples.end()); + const size_t n = lat.samples.size(); + const uint64_t qps = secs > 0 ? static_cast(n / secs) : 0; + LOG(INFO) << "RESULT phase=" << name << " reads=" << n << " qps=" << qps + << " p50=" << Percentile(lat.samples, 0.50) + << " p90=" << Percentile(lat.samples, 0.90) + << " p99=" << Percentile(lat.samples, 0.99) + << " p999=" << Percentile(lat.samples, 0.999) + << " max=" << (n ? lat.samples.back() : 0) + << " not_found=" << lat.not_found << " errors=" << lat.errors + << " (latency us)"; +} + +/** + * Read driver. Baseline phase: closed loop at FLAGS_read_concurrency. + * Mixed phase with write_read_ratio > 0: reads are additionally gated on + * credits produced by completed storm writes, holding the op mix at + * 1 read : ratio writes (reads pause when writes stall, and vice versa + * never outrun the ratio). Each completion is recorded into the phase the + * request was ISSUED in (so a request straddling a phase flip does not + * contaminate the other phase). + */ +void ReadLoop(eloqstore::EloqStore *store, + PhaseLatencies *baseline, + PhaseLatencies *mixed) +{ + moodycamel::BlockingConcurrentQueue finished; + std::vector> readers(FLAGS_read_concurrency); + std::vector idle; + idle.reserve(FLAGS_read_concurrency); + for (uint32_t i = 0; i < FLAGS_read_concurrency; i++) + { + readers[i] = std::make_unique(i); + idle.push_back(readers[i].get()); + } + + auto callback = [&finished](eloqstore::KvRequest *req) + { finished.enqueue(reinterpret_cast(req->UserData())); }; + + std::mt19937_64 rnd(12345); + auto send_req = [&](Reader *reader) + { + const uint64_t key = rnd() % FLAGS_keys_per_partition; + const uint32_t part = rnd() % FLAGS_partitions; + EncodeKey(reader->key_, key); + reader->request_.SetArgs(eloqstore::TableIdent(kTable, part), + std::string_view(reader->key_, sizeof(key))); + reader->issue_phase_ = g_phase.load(std::memory_order_relaxed); + reader->start_us_ = utils::UnixTs(); + store->ExecAsyn(&reader->request_, uint64_t(reader), callback); + }; + + size_t inflight = 0; + while (true) + { + Reader *reader; + if (finished.wait_dequeue_timed(reader, milliseconds(1))) + { + const uint64_t lat = + utils::UnixTs() - reader->start_us_; + PhaseLatencies *dst = reader->issue_phase_ == 1 ? baseline + : reader->issue_phase_ == 2 ? mixed + : nullptr; + if (dst != nullptr) + { + dst->samples.push_back(lat); + if (reader->request_.Error() == eloqstore::KvError::NotFound) + { + dst->not_found++; + } + else if (reader->request_.Error() != + eloqstore::KvError::NoError) + { + dst->errors++; + } + } + inflight--; + idle.push_back(reader); + } + + const int phase = g_phase.load(std::memory_order_relaxed); + if (phase >= 3) + { + if (inflight == 0) + { + break; + } + continue; // drain without re-issuing + } + while (!idle.empty()) + { + if (phase == 2 && FLAGS_write_read_ratio > 0 && + !TryConsumeReadCredits()) + { + break; // wait for storm writes to earn more read credits + } + send_req(idle.back()); + idle.pop_back(); + inflight++; + } + } +} + +// ------------------------------------------------------------ write storm + +struct StormWriter +{ + explicit StormWriter(uint32_t part) : part_(part) + { + } + const uint32_t part_; + eloqstore::BatchWriteRequest request_; + uint64_t next_key_{0}; + uint32_t round_{0}; + uint64_t bytes_written_{0}; + uint64_t keys_written_{0}; + uint32_t batch_keys_{0}; // keys in the currently in-flight batch + bool done_{false}; +}; + +/** + * Build the next storm batch for one partition: ascending keys where + * ((key + round) % ratio) < span. Rotating `round` shifts the surviving 40% + * every pass over the keyspace, so files written by earlier rounds are + * partially — never fully — invalidated, keeping compaction's move pipeline + * busy for the whole phase. + */ +void NextStormBatch(StormWriter &w) +{ + std::vector entries; + entries.reserve(FLAGS_storm_batch_keys); + const uint64_t ts = utils::UnixTs(); + const std::string value(FLAGS_val_size, 'S' + (w.round_ & 7)); + while (entries.size() < FLAGS_storm_batch_keys) + { + if (w.next_key_ >= FLAGS_keys_per_partition) + { + w.next_key_ = 0; + w.round_++; + if (!entries.empty()) + { + // Batch keys must stay sorted: never let a batch span the + // keyspace wrap (it would append low keys after high ones). + break; + } + } + const uint64_t key = w.next_key_++; + if ((key + w.round_) % FLAGS_storm_ratio >= FLAGS_storm_span) + { + continue; + } + entries.emplace_back( + MakeKey(key), value, ts, eloqstore::WriteOp::Upsert); + w.bytes_written_ += FLAGS_val_size + sizeof(uint64_t); + } + w.batch_keys_ = static_cast(entries.size()); + w.keys_written_ += w.batch_keys_; + w.request_.SetArgs(eloqstore::TableIdent(kTable, w.part_), + std::move(entries)); +} + +struct StormTotals +{ + uint64_t bytes{0}; + uint64_t keys{0}; +}; + +/** + * One outstanding batch write per partition (matching the engine's + * per-partition write serialization), re-issued on completion until the + * phase ends. Each completed batch grants its key count as read credits + * (see TryConsumeReadCredits). Returns total bytes/keys submitted. + */ +StormTotals StormLoop(eloqstore::EloqStore *store) +{ + moodycamel::BlockingConcurrentQueue finished; + std::vector> writers(FLAGS_partitions); + for (uint32_t i = 0; i < FLAGS_partitions; i++) + { + writers[i] = std::make_unique(i); + } + auto callback = [&finished](eloqstore::KvRequest *req) + { finished.enqueue(reinterpret_cast(req->UserData())); }; + + for (auto &w : writers) + { + NextStormBatch(*w); + store->ExecAsyn(&w->request_, uint64_t(w.get()), callback); + } + size_t inflight = writers.size(); + while (inflight > 0) + { + StormWriter *w; + finished.wait_dequeue(w); + CHECK(w->request_.Error() == eloqstore::KvError::NoError) + << "storm write failed: " + << eloqstore::ErrorString(w->request_.Error()); + g_read_credits.fetch_add(w->batch_keys_, std::memory_order_relaxed); + if (g_phase.load(std::memory_order_relaxed) >= 3) + { + inflight--; + continue; + } + NextStormBatch(*w); + store->ExecAsyn(&w->request_, uint64_t(w), callback); + } + StormTotals total; + for (auto &w : writers) + { + total.bytes += w->bytes_written_; + total.keys += w->keys_written_; + } + return total; +} + +// ------------------------------------------------------------------- load + +void Load(eloqstore::EloqStore *store) +{ + const std::string value(FLAGS_val_size, 'L'); + const uint64_t ts = utils::UnixTs(); + for (uint32_t part = 0; part < FLAGS_partitions; part++) + { + for (uint64_t base = 0; base < FLAGS_keys_per_partition; + base += FLAGS_storm_batch_keys) + { + const uint64_t end = std::min( + base + FLAGS_storm_batch_keys, FLAGS_keys_per_partition); + std::vector entries; + entries.reserve(end - base); + for (uint64_t key = base; key < end; key++) + { + entries.emplace_back( + MakeKey(key), value, ts, eloqstore::WriteOp::Upsert); + } + eloqstore::BatchWriteRequest req; + req.SetArgs(eloqstore::TableIdent(kTable, part), + std::move(entries)); + store->ExecSync(&req); + CHECK(req.Error() == eloqstore::KvError::NoError) + << "load failed: " << eloqstore::ErrorString(req.Error()); + } + LOG(INFO) << "loaded partition " << part << " (" + << FLAGS_keys_per_partition << " keys)"; + } +} + +// ------------------------------------------------------------------ stats + +void ReportQosDelta(const char *name, + const eloqstore::IoQosStats &begin, + const eloqstore::IoQosStats &end, + size_t shard, + double secs) +{ + auto d = [](uint64_t b, uint64_t e) { return e - b; }; + // Store-issued page IO in MB/s over the phase: every page metered by the + // budgets (user data + compaction relocations + index pages). This is + // the device-facing rate to compare against the fio calibration curve — + // unlike the workload's logical write MB/s, it includes background + // amplification (and excludes only manifest/fdatasync/segment IO). + auto mbps = [&](const eloqstore::IoQosStats::Budget &b, + const eloqstore::IoQosStats::Budget &e) + { + return secs > 0 ? (d(b.admitted_pages_, e.admitted_pages_) * 4096) / + (secs * (1 << 20)) + : 0.0; + }; + LOG(INFO) << "RESULT qos phase=" << name << " shard=" << shard + << " read_hwm=" << end.read_.high_watermark_ + << " read_blocked=" << d(begin.read_.blocked_count_, + end.read_.blocked_count_) + << " read_blocked_us=" << d(begin.read_.blocked_us_, + end.read_.blocked_us_) + << " read_page_mbps=" << mbps(begin.read_, end.read_) + << " bg_read_hwm=" << end.bg_read_.high_watermark_ + << " bg_read_blocked=" << d(begin.bg_read_.blocked_count_, + end.bg_read_.blocked_count_) + << " bg_read_blocked_us=" << d(begin.bg_read_.blocked_us_, + end.bg_read_.blocked_us_) + << " bg_read_page_mbps=" << mbps(begin.bg_read_, end.bg_read_) + << " write_hwm=" << end.write_.high_watermark_ + << " write_blocked=" << d(begin.write_.blocked_count_, + end.write_.blocked_count_) + << " write_page_mbps=" << mbps(begin.write_, end.write_) + << " fdatasync=" << d(begin.fdatasync_count_, + end.fdatasync_count_) + << " fdatasync_us=" << d(begin.fdatasync_us_, + end.fdatasync_us_); +} + +} // namespace + +int main(int argc, char *argv[]) +{ + google::ParseCommandLineFlags(&argc, &argv, true); + CHECK_GT(FLAGS_partitions, 0u); + CHECK_GT(FLAGS_read_concurrency, 0u); + CHECK_GT(FLAGS_storm_span, 0u); + CHECK_GT(FLAGS_storm_ratio, FLAGS_storm_span) + << "storm must be a PARTIAL overwrite (span < ratio); a full " + "overwrite leaves files 100% dead and compaction generates no " + "read traffic"; + + eloqstore::KvOptions options; + if (int res = options.LoadFromIni(FLAGS_kvoptions.c_str()); res != 0) + { + LOG(FATAL) << "Failed to parse " << FLAGS_kvoptions << " at " << res; + } + LOG(INFO) << "QoS knobs: max_inflight_read=" << options.max_inflight_read + << " bg_read_ratio=" << options.bg_read_ratio + << " max_inflight_write=" << options.max_inflight_write; + + eloqstore::EloqStore store(options); + if (auto err = store.Start("main", 0); err != eloqstore::KvError::NoError) + { + LOG(FATAL) << "Failed to start store: " << eloqstore::ErrorString(err); + } + + if (FLAGS_load) + { + Load(&store); + } + + const size_t num_shards = store.Options().num_threads; + std::vector qos_start(num_shards); + std::vector qos_mid(num_shards); + std::vector qos_end(num_shards); + + PhaseLatencies baseline, storm_lat; + + // Baseline phase: reads only. + for (size_t s = 0; s < num_shards; s++) + { + qos_start[s] = store.GetIoQosStats(s); + } + g_phase.store(1, std::memory_order_relaxed); + std::thread read_thd(ReadLoop, &store, &baseline, &storm_lat); + std::this_thread::sleep_for(seconds(FLAGS_baseline_secs)); + + // Storm phase: reads + rotating partial-overwrite write storm. + for (size_t s = 0; s < num_shards; s++) + { + qos_mid[s] = store.GetIoQosStats(s); + } + g_phase.store(2, std::memory_order_relaxed); + StormTotals storm_totals; + std::thread storm_thd([&] { storm_totals = StormLoop(&store); }); + std::this_thread::sleep_for(seconds(FLAGS_storm_secs)); + + g_phase.store(3, std::memory_order_relaxed); + storm_thd.join(); + read_thd.join(); + for (size_t s = 0; s < num_shards; s++) + { + qos_end[s] = store.GetIoQosStats(s); + } + + ReportPhase("baseline", baseline, FLAGS_baseline_secs); + ReportPhase("mixed", storm_lat, FLAGS_storm_secs); + const uint64_t read_ops = storm_lat.samples.size(); + const double achieved_write_pct = + storm_totals.keys + read_ops > 0 + ? 100.0 * storm_totals.keys / (storm_totals.keys + read_ops) + : 0.0; + LOG(INFO) << "RESULT mixed write_mb_per_sec=" + << (storm_totals.bytes >> 20) / + std::max(1, FLAGS_storm_secs) + << " write_mb=" << (storm_totals.bytes >> 20) + << " write_key_ops=" << storm_totals.keys + << " read_ops=" << read_ops << " achieved_write_pct=" + << static_cast(achieved_write_pct + 0.5); + for (size_t s = 0; s < num_shards; s++) + { + ReportQosDelta( + "baseline", qos_start[s], qos_mid[s], s, FLAGS_baseline_secs); + ReportQosDelta("mixed", qos_mid[s], qos_end[s], s, FLAGS_storm_secs); + } + + store.Stop(); + return 0; +} diff --git a/benchmark/opts_interference.ini b/benchmark/opts_interference.ini new file mode 100644 index 000000000..5669fcfb5 --- /dev/null +++ b/benchmark/opts_interference.ini @@ -0,0 +1,30 @@ +# EloqStore options for interference_bench (docs/design/io_qos.md commit 3). +# +# Append mode with small (8MB) data files so the rotating partial-overwrite +# storm pushes per-file space amplification past file_amplify_factor quickly, +# keeping compaction's move pipeline (background ReadPages bursts) busy. +# +# Sweep the QoS knobs below (see io_qos_impl_plan.md "Performance +# acceptance"): max_inflight_read in {64,128,256,512}, bg_read_ratio in +# {10,25,50}, max_inflight_write in {256,512,1024}. + +# Note: run interference_bench with --partitions >= num_threads, or some +# shards sit idle and their RESULT qos lines read zero. + +[run] +num_threads = 4 +buffer_pool_size = 400MB +fd_limit = 5000 +num_retained_archives = 0 +skip_verify_checksum = true + +# --- IO QoS knobs under test --- +max_inflight_read = 32 +bg_read_ratio = 25 +max_inflight_write = 512 + +[permanent] +store_path = /tmp/eloqstore_interference +data_page_size = 4KB +data_file_size = 8MB +data_append_mode = true diff --git a/docs/architecture/02-runtime-and-lifecycle.md b/docs/architecture/02-runtime-and-lifecycle.md index 171dd5c7e..eb3f628f3 100644 --- a/docs/architecture/02-runtime-and-lifecycle.md +++ b/docs/architecture/02-runtime-and-lifecycle.md @@ -105,9 +105,16 @@ state: - **Topology**: `num_threads` (shards), `fd_limit`, `io_queue_size`, `buffer_pool_size` (index-page cache per shard; data pages share it only if `enable_data_page_cache`), `root_meta_cache_size` (global RootMeta LRU). -- **Write shaping**: `max_write_batch_pages`, `max_write_concurrency`, +- **Write shaping**: `max_write_concurrency`, `write_buffer_size`/`write_buffer_ratio` (append-mode aggregation), `manifest_limit` (snapshot-rotation threshold). + (`max_write_batch_pages` is deprecated and ignored — superseded by the + `max_inflight_write` IO budget, doc 07.) +- **IO QoS** (doc 07, `docs/design/io_qos.md`): `max_inflight_read` + (device-calibrated read queue-depth cap), `bg_read_ratio` (background + read sub-budget, the tail-predictability policy knob), + `max_inflight_write` (write queue-depth cap; also sizes the write + request pools). - **Cloud**: `cloud_provider/endpoint/region/keys`, `local_space_limit` + `reserve_space_ratio` (cache budget), `max_cloud_concurrency`, `cloud_request_threads`, `allow_reuse_local_caches`, `prewarm_cloud_cache`. diff --git a/docs/architecture/04-execution-model.md b/docs/architecture/04-execution-model.md index f35421186..93134b6f9 100644 --- a/docs/architecture/04-execution-model.md +++ b/docs/architecture/04-execution-model.md @@ -59,6 +59,18 @@ Scheduling primitives: - `WaitingZone` / `WaitingSeat` / `Mutex` — intra-shard wait lists (no real locks; they park/wake coroutines). Used for FD open/close exclusion, pool exhaustion waits, upload completion, etc. +- IO-budget waits (`IoBudget::Acquire`, `async_io_manager.h`) — tasks park on + a budget's `WaitingZone` when admitting their page IO would exceed the + shard's in-flight read/write cap (`max_inflight_read` / + `max_inflight_write`); `PollComplete` releases per CQE and wakes waiters, + so release never depends on the blocked task being scheduled. Background + tasks (`KvTask::IsBackground()`: BatchWrite, BackgroundWrite, EvictFile, + Prewarm) are additionally confined to a read sub-budget (`bg_read_ratio`) + and wait on a separate FIFO zone. While background waiters queue, their + unused sub-budget is reserved from new foreground admissions (neither + class can starve the other); release wakes background first and forwards + unused wake credits to foreground. See `docs/design/io_qos.md` (M1/M2); + the acquire order is FD/mutex → pools/buffers → budget → SQE. `TaskManager` keeps one free-list pool per task type (`BatchWriteTask`, `BackgroundWrite`, `ReadTask`, `ScanTask`, `ListObjectTask`, diff --git a/docs/architecture/07-io-stack.md b/docs/architecture/07-io-stack.md index 89bfb23a7..df7903700 100644 --- a/docs/architecture/07-io-stack.md +++ b/docs/architecture/07-io-stack.md @@ -37,6 +37,31 @@ Responsibilities: - **Page I/O** — `ReadPage`/`ReadPages` (batched, into pool buffers, fixed reads when the buffer is registered), `WritePage`. `ConvFilePageId` splits a `FilePageId` into `(file_id, offset)` by `pages_per_file_shift`. +- **In-flight page-IO budgets** (`IoBudget`, see `docs/design/io_qos.md` + M1/M2) — two per-shard counters in 4KB-page units with independent caps: + `read_budget_` (`max_inflight_read`; `ReadPage`/`ReadPages`, per-page + acquisition so a batch larger than the cap cannot deadlock) and + `write_budget_` (`max_inflight_write`; `WritePage` cost 1, + `SubmitMergedWrite` cost `bytes / data_page_size`). Budget is acquired + immediately before SQE prep and released per CQE in `PollComplete`, which + distinguishes budgeted page reads from metadata ops via the + `KvTaskPageRead`/`BaseReqPageRead` user-data types. A cap of 0 disables a + budget; a single request costlier than the cap is admitted alone once the + budget drains. Metadata, manifest, and segment IO are exempt. + The read budget carries a **background sub-budget** (`bg_read_ratio` + percent of `max_inflight_read`): page reads from tasks where + `KvTask::IsBackground()` (BatchWrite, BackgroundWrite, EvictFile, Prewarm) + are additionally bounded by it, so compaction/GC/batch-write read bursts + cannot crowd foreground point reads out of the device queue. Foreground + may use the entire read budget while background has no queued demand; + once background waiters exist their unused sub-budget is reserved from + new foreground admissions, so neither class can starve the other. Each + class waits on its own FIFO zone; release wakes background first and + forwards unused wake credits to foreground. The write budget has no + split — all page writes come from write tasks, i.e. background. + `GetIoQosStats()` (also surfaced as `EloqStore::GetIoQosStats(shard_id)`) + exposes in-flight/high-watermark/blocked counters (read, bg-read slice, + write) plus write-path fdatasync count and latency. - **FD cache** — `LruFD` per (partition, `TypedFileId`), doubly-linked LRU bounded by the shard's fd budget; `EvictFD` closes idle descriptors. Open/close exclusion per FD via a coroutine `Mutex`. Data files open with diff --git a/docs/architecture/08-data-lifecycle.md b/docs/architecture/08-data-lifecycle.md index 7b4606ec0..fd3fb754c 100644 --- a/docs/architecture/08-data-lifecycle.md +++ b/docs/architecture/08-data-lifecycle.md @@ -67,6 +67,12 @@ user writes. Scheduled via shard pending-sets (`AddPendingCompact/TTL/FileGc/LocalGc`) which enqueue the embedded singleton requests in each `PendingWriteQueue`. +All of these run as background tasks (`KvTask::IsBackground()`), so their +data-page reads — compaction move batches in particular — are charged +against the read budget's background sub-budget (`bg_read_ratio`, +doc 07 / `docs/design/io_qos.md` M2) and cannot crowd out foreground reads +at the device; their page writes are bounded by `max_inflight_write`. + - **Compaction** (`Compact()`, append mode only): one `MakeCowRoot`; rewrite pass over under-utilized data files (live pages re-written to the tail, per-file utilization re-checked against `file_amplify_factor`), then over diff --git a/docs/design/io_qos.md b/docs/design/io_qos.md new file mode 100644 index 000000000..b84346fd9 --- /dev/null +++ b/docs/design/io_qos.md @@ -0,0 +1,388 @@ +# EloqStore IO QoS: Foreground/Background IO Isolation + +## Summary + +EloqStore runs foreground tasks (ReadTask, ScanTask) and background tasks +(BatchWriteTask, BackgroundWrite/compaction, file GC) as coroutines on a +per-shard thread. CPU-side prioritization works: background tasks yield to a +low-priority ready queue and the scheduler bounds low-priority slices to +`max_processing_time_microseconds`. However, experiments show background tasks +still significantly degrade foreground read latency. + +PR #455 (`perf(compaction/gc)`) addressed the CPU half of this problem: the +time-budgeted cooperative yield (`MaybeYield()` / +`eloqstore_yield_budget_us`, default 20µs) bounds how long any single +compaction/GC coroutine segment holds the worker thread, and GC unlink/close +SQE preparation is chunked into 128-op batches. What remains — and what this +document addresses — is that **CPU priority is lost at the IO boundary**. +Within one 20µs slice a background task can still burst up to 128 page reads +(compaction move batches, `DoCompactDataFile`) plus writes into the io_uring +ring, and nothing distinguishes those requests from foreground reads at the +ring or device level. Reads are not budgeted at all; writes are budgeted only +per-task. Yielding controls when background IO is *issued*, not how much of +it queues at the device once issued. + +This document proposes three per-shard mechanisms: + +- **M1**: per-shard caps on in-flight page IO, with **separate caps for + reads and writes** — they are different device resources and must be + tunable independently. +- **M2**: foreground/background classification with a background sub-budget + on the read cap (all page writes come from write tasks, i.e. background, + so the write cap needs no split). +- **M3**: a bytes/sec rate limiter on background writes (follow-up, driven by + measurement; complements the write cap, which bounds queue depth but not + sustained throughput). + +All caps and limits in this document are **per shard**, consistent with the +rest of EloqStore. + +Related maintained documentation: `docs/architecture/04-execution-model.md` +(scheduler, ready queues, yield discipline), `07-io-stack.md` (IouringMgr, +page/segment IO, pools), `08-data-lifecycle.md` (compaction/GC cadence). +Implementing this design requires updating those docs in the same change +(see CLAUDE.md). + +## Current Mechanisms and Gaps + +### What exists today + +| Mechanism | Location | Scope | +|---|---|---| +| Two ready queues (high / low priority), 400µs low-priority slice | `Shard::ExecuteReadyTasks`, `KvTask::YieldToLowPQ` | CPU only | +| Time-budgeted cooperative yield: `MaybeYield()` once a coroutine segment exceeds `eloqstore_yield_budget_us` (20µs) — added by #455 | `MaybeYield`, `Shard::CurResumeElapsedUs` | CPU only | +| GC unlink/close SQE prep chunked into 128-op batches, `WaitIo` per chunk — added by #455 | `IouringMgr::CloseFiles` / `DeleteFiles` | Metadata IO burst size | +| Per-write-task in-flight cap: `WaitWrite()` when `inflight_io_ >= max_write_batch_pages` (32) | `WriteTask::WritePage` | Writes only, per task | +| `WriteReqPool` sized by `max_inflight_write` (default 32768) | `IouringMgr` | Writes, effectively unbounded | +| Segment compaction: ≤ `max_segments_batch` (8) in-flight 256KB segments per batch, buffers shared with foreground via `GlobalRegisteredMemory`; yields every `segment_compact_yield_every` segments | `BackgroundWrite::DoCompactSegmentFile` | Segment IO, per task | +| SQ ring capacity `io_queue_size` (4096); `GetSQE` blocks on full ring | `IouringMgr::GetSQE`, `waiting_sqe_` | Backpressure, not QoS | +| `max_write_concurrency` bounds concurrent write tasks (incl. compaction: `GetBackgroundWrite` returns null at the limit) | `TaskManager` | Default unlimited in local mode | +| Cloud slots (`max_cloud_concurrency`), cloud buffer pool | `CloudStoreMgr` | Cloud HTTP, not disk | + +### Gaps + +1. **Reads have no budget anywhere.** Page compaction + (`DoCompactDataFile`) issues bursts of up to `max_read_pages_batch` (128) + page reads per move batch through the same `ReadPages` path as foreground + reads. +2. **No shard-global in-flight IO counter.** Only per-task `inflight_io_` and + `prepared_sqe_` exist. The effective global bounds (4096 SQEs, 32K + in-flight writes) are far beyond the queue depth at which NVMe read + latency degrades. +3. **No FG/BG tagging of IO.** SQEs are indistinguishable once submitted; + `sqe->ioprio` is never set. A background task scheduled for one 20µs + slice can leave 128+ requests queued at the device ahead of every + subsequent foreground read — the yield discipline of #455 bounds CPU + occupation per slice, not the IO submitted within it. + +## Device-Level Rationale + +EloqStore's pattern in append mode is **sequential writes vs. random reads**, +which makes the interference mechanism clean: + +- Reads and writes are asymmetric on NVMe. A 4KB write is acknowledged from + the controller buffer (~10–20µs); a 4KB read must touch NAND (~60–100µs). + The deferred cost of writes is NAND program time (~1–3ms per TLC program) + and die occupancy. +- **Die collisions dominate.** A random read landing on a die that is + mid-program waits for it (unless the drive supports program-suspend). To + first order, if background write throughput is fraction ρ of the device's + sustained write bandwidth, then P(read collides) ≈ ρ, and collided reads + pay ~0.5–1.5ms. Median read latency stays flat; tail latency degrades + roughly linearly with write **bytes/sec** — not with in-flight write count. +- Sequential appends keep drive-internal write amplification near 1 + (whole-file GC invalidates large contiguous regions), so there is no + hidden GC multiplier — provided deleted file space is actually TRIMmed. + +Consequences for the design: + +- An in-flight cap (M1/M2) bounds burst queueing at the ring/device, but the + control that maps to the physics of sustained interference is a **write + bytes/sec budget** (M3). +- **Pace, don't burst**: the same average MB/s issued as evenly spaced 1MB + merged writes produces better read tails than periodic multi-MB bursts. + Keep the 1MB merge unit; throttle frequency, not size. + +## Design + +### M1: Per-shard in-flight page-IO caps, reads and writes separate + +Two per-shard counters of in-flight **page** IO (in 4KB-page units), with +independent caps, enforced at the page-IO entry points of `IouringMgr`. +Reads and writes are deliberately **not** mixed in one budget: they are +different device resources (reads need deep queue depth for IOPS; writes +saturate bandwidth at shallow depth and their interference tracks sustained +bytes/sec), so a shared cap would couple two knobs that must be tuned +separately. + +- Read budget (`inflight_read_pages_` ≤ `max_inflight_read`): + `ReadPage` / `ReadPages`, cost 1 per page. +- Write budget (`inflight_write_pages_` ≤ `max_inflight_write`, option + redefined — see Interaction with Existing Knobs): + `WritePage`, cost 1; `SubmitMergedWrite`, cost = bytes / data_page_size + (a 1MB merged write counts as 256 pages), so the cap means the same thing + in append and non-append mode. +Segment IO (`ReadSegments` / `WriteSegments`, zero-copy large values) is +**out of scope** — see Non-Goals. + +Metadata operations (open, statx, rename, unlink, mkdir) are exempt; their +burst size is already bounded by the 128-op chunking from #455. +`fdatasync` is not counted initially but is instrumented (see Evaluation). + +Enforcement follows the existing idiom, per class (read or write). The +in-flight counter holds only previously admitted, not-yet-completed IO; each +request is counted exactly once, from admission to completion. For a new +request of size `cost` (page-units, not yet counted): + +``` +acquire(class, cost): // at the page-IO entry point + while inflight[class] + cost > cap[class]: + wait on waiting_zone[class] (FIFO) + inflight[class] += cost // admitted; prep SQE(s) and submit + +release(class, cost): // in PollComplete, on completion + inflight[class] -= cost + wake waiters of that class +``` + +Blocking-on-acquire is preferred over yield-and-retry: it is cheaper and +guarantees FIFO fairness. + +Rules: + +- Acquire the budget immediately before SQE preparation, with **no yield + points between acquire and submit**. +- Release happens in `PollComplete`, which runs regardless of task + scheduling, so budget release never depends on the blocked task running — + no deadlock. Paths that hold budget and then block on another pool (e.g. + write buffer pool) must acquire in a consistent order; audit during + implementation. + +### M2: Foreground/background classification and BG sub-budget + +Every page IO is issued from a `KvTask` (`ThdTask()`), so classification is a +task-type predicate. Add `KvTask::IsBackground()`: + +- Background: `BatchWrite`, `BackgroundWrite`, `EvictFile`, `Prewarm`. +- Foreground: `Read`, `Scan`, `ListObject`, `ListStandbyPartition`, `Reopen`. + +Do **not** reuse `ReadOnly()` — `EvictFile` and `Prewarm` are read-only but +background. + +The FG/BG split only applies to the **read** budget. All page writes are +issued by write tasks (`BatchWrite`, `BackgroundWrite`), which are background +by the classification above — so the write cap of M1 *is* the background +write budget, and needs no further split. (If a genuinely foreground write +class ever appears, split the write budget then.) + +Read budget structure: + +- Foreground reads: `inflight_read_pages_ ≤ max_inflight_read`. +- Background reads (compaction move batches, batch-write tree-traversal + reads, GC/upload-path reads): additionally + `bg_inflight_read_pages_ ≤ bg_read_limit` (a fraction of + `max_inflight_read`, e.g. 25%). + +Background never exceeds its sub-budget. Foreground can consume the entire +read budget **while background has no queued demand**; once background +waiters exist, their unused entitlement (`bg_read_limit − bg_inflight`) is +reserved — new foreground admissions leave it alone. Without the +reservation, sustained foreground saturation would starve background +forever (every freed unit would be re-acquired by a foreground waiter and +compaction could never bootstrap its share), letting space amplification +grow unboundedly. With it, the ratio is genuinely maintained under +contention: background ramps to its slice, foreground keeps the rest, and +foreground reclaims the whole budget the moment background demand drains. + +Two read `WaitingZone`s (one per class, FIFO within class). On release, +background waiters are woken first while the sub-budget has room (those +units are reserved for them anyway); unused wake credits are forwarded to +foreground (`WaitingZone::WakeN` returns the number actually woken). + +The ratio starts **static**. An adaptive policy (shrink `bg_read_limit` +toward a floor when foreground waiters exist, grow toward the full budget +when the shard is idle) is a possible follow-up once counters justify it. + +### M3: Background write rate limiter (follow-up) + +A per-shard token bucket in bytes, refilled at `bg_write_rate_limit` bytes/sec: + +- `WritePage` / `SubmitMergedWrite` from background tasks debit the bucket + and block on a `WaitingZone` when empty. +- Bucket capacity is small (a few refill intervals worth) so a full bucket + cannot discharge as a burst; this paces merged writes evenly. +- 0 = disabled (default). Enable after the calibration sweep (below) shows + the tail-latency knee for the target device. + +M3 is deferred until experiments confirm that in-flight capping alone (M1+M2) +is insufficient — which the die-collision model predicts for sustained +compaction, since interference tracks bytes/sec rather than queue depth. + +### New options (per shard) + +``` +uint32_t max_inflight_read = 32; // pages; 0 = disabled +uint32_t bg_read_ratio = 25; // percent of max_inflight_read +uint32_t max_inflight_write = 512; // pages; REDEFINED existing option + // (was 32768, pool sizing only) +uint64_t bg_write_rate_limit = 0; // bytes/sec; 0 = disabled (M3) +``` + +max_inflight_read = 32 and bg_read_ratio = 25 were set from the WSL +interference sweeps (2026-07-03): total caps 32–256 were indistinguishable +at fixed bg_cap, 32 matched that box's BDP and gave the campaign's best +p50/p999, and bg_cap 8 (25% of 32) sat exactly at the measured background +demand line — tail protection at zero write-throughput cost (the +write-throughput knee appeared only at bg_cap 4). Real-device calibration +(the QD sweep below) should re-derive max_inflight_read per device; the +ratio is policy and should transfer. max_inflight_write's real default +still awaits commit 4. + +### Sizing contract: device knob × policy knob + +The two read-side options deliberately live at different levels: + +- **`max_inflight_read` is device calibration.** Size it from the device's + bandwidth-delay product: `c × max_random_read_IOPS × t_read(unloaded) / + num_threads`, with c ≈ 2–4 for die-imbalance and burst headroom. Below + the BDP the cap costs read throughput; far above it the cap stops + representing the device queue and the ratio contract below degrades. + Validation signal: foreground `read_blocked` should stay ≈ 0 under + representative load — nonzero means undersized. +- **`bg_read_ratio` is policy, and is deliberately a ratio.** A foreground + read's worst-case queueing behind background is `bg_cap / IOPS`; with the + cap BDP-sized this equals `ratio × c × t_read` — the device's IOPS + cancels. Relative tail inflation (tail as a multiple of the device's own + base latency) is therefore a function of the ratio alone: a slower device + serves proportionally slower reads with the **same tail-to-median shape**. + That predictability is the operator-facing contract, it survives hardware + changes without retuning, and it is why the option must not be an + absolute page count. +- Background demand scales with the same device parameters in device-bound + deployments, so the ratio budget and the demand shrink and grow together. + If measured demand persistently exceeds the budget + (`bg_read_blocked_us` ≈ wall time), compaction is being deferred: the + remedy is raising `max_inflight_read` (the device has headroom the + calibration missed), not the ratio — raising the ratio spends the tail + contract. + +## Interaction with Existing Knobs + +- **`io_queue_size` (SQ ring) stays.** With both caps ≪ 4096, tasks + should essentially never block in `GetSQE`; the ring cap becomes a sanity + bound rather than a throttle. +- **`max_write_batch_pages` throttle retires.** *(Implemented, plan commit + 4.)* The `inflight_io_ >= cap → WaitWrite()` branch in + `WriteTask::WritePage` is subsumed by M1/M2 and is strictly worse (it + drains to zero and restarts, producing a sawtooth; budget acquisition + keeps a steady level). Keep: the CPU yields between page builds + (`YieldToLowPQ()` / `MaybeYield()` — the #455 time-budgeted discipline is + orthogonal and stays), and the terminal `WaitWrite()` before + `UpdateMeta`/`SyncData` (error collection and durability ordering). + One consequence surfaced by tests: with `enable_data_page_cache`, a write + task's write-promotion pins on cached pages are now bounded by + `max_inflight_write` rather than by the per-task drain, so pathologically + small buffer pools must account for in-flight-write pins (production + pools dwarf the ≤ max_inflight_write pages of pins; only tests noticed). +- **`max_inflight_write` is redefined, not retired.** It becomes the M1 + write cap (page units, default dropped from 32768 to a few hundred). + `WriteReqPool` stays sized to it; in-flight writes can never exceed it, so + the pool bound and the QoS bound coincide. Note this is a behavioral + change for deployments that set the old option explicitly. +- **`max_write_concurrency` is no longer an IO-QoS knob.** Note it is + enforced **per shard** (each shard's `TaskManager` counts its own active + write tasks against the store-wide option value), so the device-wide task + bound is `max_write_concurrency × num_threads`. It remains as a + cross-partition admission/memory bound (coroutine stacks, 1MB write + buffers, CoW roots with retained mapping snapshots — per-partition writes + are already serialized by `PendingWriteQueue`). Local-mode guidance: + a small constant (k = 2–4). Rationale: + - k = 1 forfeits intra-shard pipelining: each batch's serial IO tail + (final drain + fdatasync + manifest append) becomes dead time for that + shard's queued partitions. k = 2 recovers most of it. + - With 4–8 shards per device, device-level parallelism, utilization, and + stream mixing are set by shard count, not k; k is purely an intra-shard + pipelining knob. Per-shard k also provides skew robustness when write + load concentrates on few shards. + - Small k additionally staggers fdatasync phases, avoiding fsync bursts. + - Cloud mode keeps `max_cloud_concurrency`-scale values: the terminal + stall is an upload (tens of ms), so higher k is required to keep the + shard's write pipeline busy. + +## Multi-Shard Considerations + +All budgets are per shard, but the device is shared by all shards +(typically 4–8 per NVMe device): + +- Size per-shard budgets as device budget / num_threads. Static division is + safe (aggregate background pressure is bounded regardless of which shards + are busy) but conservative: a lone compacting shard gets 1/S of the + background budget. Acceptable initially. +- Follow-up: a shared global token bucket (atomic counter) for the + background budget. Cross-shard wake-up is handled by each shard's + `WorkLoop` re-checking the bucket once per round and waking its local + waiters — no cross-thread coroutine resumption needed. +- Cloud mode: NIC bandwidth is shared between background uploads and + foreground cache-miss downloads; the same FG/BG discipline should + eventually apply to cloud slots. Also note upload-path disk reads + (`ReadFilePrefix`) are issued by background write tasks and are therefore + automatically counted against the BG disk budget — remember this when + sizing it in cloud mode. + +## Non-Goals + +- **Segment IO (zero-copy large values) is not budgeted.** Segments exist to + serve very large values, a workload where throughput matters rather than + tail latency and fairness, so FG/BG isolation is not a goal there. + Segment IO is neither counted against `max_inflight_io` nor classified; + segment compaction keeps its existing bounds (`max_segments_batch` + in-flight segments per batch via the shared `GlobalRegisteredMemory` pool, + `segment_compact_yield_every`). Revisit only if large-value traffic is + ever mixed with latency-sensitive point reads on the same store. +- No `sqe->ioprio` reliance: only honored by mq-deadline/BFQ schedulers + (NVMe defaults to `none`). May be added as a one-line supplement, never as + the mechanism. +- No deferred BG staging queue (strict priority instead of a ratio): more + complexity; revisit only if M1–M3 prove insufficient. +- No cross-shard coordination in the first iteration. + +## Observability + +Export per-shard counters from day one; tuning must be measurement-driven: + +- Current and high-watermark in-flight pages, split FG/BG, read/write. +- Cumulative blocked-time and block counts per class (budget wait). +- Background write bytes/sec (actual, vs. limit when M3 lands). +- fdatasync count and latency histogram. + +## Evaluation Plan + +0. **Re-baseline on top of #455.** The interference experiments that + motivated this design predate the time-budgeted yield and GC chunking. + Re-run the read-vs-compaction benchmark first to quantify how much + foreground tail latency remains attributable to device-side queueing + (this design's target) vs. worker-thread stalls (already fixed). The + remaining gap sets the success criterion for M1+M2. +1. **Device calibration sweep** (per target device, preconditioned drive): + fixed 4KB random-read load at target QD, step up sequential write MB/s, + plot read p99/p99.9 vs. write rate. The knee is the background budget. + Expect a near-linear tail-latency curve for the sequential-write pattern. + Run long enough to exhaust any SLC cache so consumer-drive folding + behavior does not confound results. +2. **Isolate fdatasync impact**: repeat the interference experiment with + sync counts instrumented; if fsync stalls dominate, sync spacing/batching + matters more than page-IO budgets and no in-flight cap will fix it. +3. **Verify TRIM**: confirm the filesystem issues discards for GC-deleted + files (`discard` mount option or periodic `fstrim`); otherwise the + WAF ≈ 1 assumption silently breaks on aged drives. +4. **Staged rollout**: + - Land M1+M2 with existing throttles left in place (loosened). + - Re-run the read-vs-compaction benchmark; confirm the new mechanism + alone protects foreground p99. + - Remove the `max_write_batch_pages` throttle and drop the + `max_inflight_write` default to its new QoS value in a follow-up + commit, so a regression identifies which mechanism was load-bearing. + - Sweep `max_write_concurrency` ∈ {1, 2, 4, 8} in local mode; pick the + throughput knee (expected at 2–4). + - Evaluate M3 against the calibration curve; enable if M1+M2 leave a + sustained-write tail-latency gap. diff --git a/docs/design/io_qos_impl_plan.md b/docs/design/io_qos_impl_plan.md new file mode 100644 index 000000000..be084d627 --- /dev/null +++ b/docs/design/io_qos_impl_plan.md @@ -0,0 +1,337 @@ +# IO QoS: Implementation and Testing Plan + +Companion to `io_qos.md` (the design). This document maps M1/M2 (and the M3 +follow-up) onto concrete code changes, commit sequence, and tests. File and +symbol references are as of `main` @ 8f5f699. + +## Commit sequence + +Five commits, each independently landable and revertible. Docs +(`docs/architecture/04-execution-model.md`, `07-io-stack.md`, +`08-data-lifecycle.md`) are updated in the same commit as the behavior they +describe (CLAUDE.md requirement). + +| # | Content | Risk | +|---|---|---| +| 1 | M1: budget state, acquire/release plumbing, options, stats getter | Core; behavior-neutral at default caps until tightened | +| 2 | M2: `IsBackground()`, BG read sub-budget, wake policy | Small delta on top of 1 | +| 3 | Interference benchmark + fio calibration script | Test-only | +| 4 | Retire `max_write_batch_pages` throttle; drop `max_inflight_write` default to QoS value | Behavioral; gated on benchmark results | +| 5 | M3 rate limiter (deferred; only if step-4 benchmarks show a sustained-write gap) | Optional | + +## Commit 1 — M1: in-flight page-IO budgets + +> **Status: implemented** (2026-07-02). Deviations from the plan below: +> `IoQosStats` includes fdatasync counters; the oversized-request escape +> (`Acquire` admits a request costlier than the cap once the budget drains) +> was added so small configured caps cannot deadlock against ~1MB merged +> writes; tests use `REQUIRE` (glog's `CHECK` macro shadows Catch2's). + +### State (in `IouringMgr`, per shard; `MemStoreMgr` untouched) + +```cpp +// include/async_io_manager.h +class IoBudget +{ +public: + void Acquire(uint32_t cost); // blocks ThdTask() on waiting_ (FIFO) + void Release(uint32_t cost); // decrement + WakeN; called by PollComplete + // stats: current_, high_watermark_, total_blocked_count_, ... +private: + uint32_t inflight_{0}; + uint32_t cap_{0}; // 0 = disabled (Acquire is a no-op) + WaitingZone waiting_; +}; + +IoBudget read_budget_; // cap = max_inflight_read +IoBudget write_budget_; // cap = max_inflight_write (redefined) +``` + +Follow the trailing-underscore member convention. `Acquire` loops +`while (inflight_ + cost > cap_) waiting_.Wait(ThdTask());` — the counter +holds only admitted, not-yet-completed IO (see design doc pseudocode). + +### Release-side classification: new `UserDataType`s + +`PollComplete` must know each CQE's class and cost. `EncodeUserData` packs +the type into the low 8 bits (`async_io_manager.cpp:1519`), so there is +ample room. Current types: `KvTask`, `BaseReq`, `WriteReq`, +`MergedWriteReq`. Add: + +- `KvTaskPageRead` — single-page read issued via the `KvTask` path + (`IouringMgr::ReadPage`). Cost 1. Handled identically to `KvTask` in + `PollComplete`, plus `read_budget_.Release(1)`. +- `BaseReqPageRead` — per-page read in a batch (`IouringMgr::ReadPages`). + Handled identically to `BaseReq`, plus `read_budget_.Release(1)`. + +Metadata ops keep `KvTask`/`BaseReq` and are never charged. Write costs are +already derivable: `WriteReq` = 1 page; `MergedWriteReq` = `bytes_ / +data_page_size` (round up; assert page alignment). + +FG/BG at release time (needed by commit 2) comes from the task pointer every +req type already carries: `task->IsBackground()`. + +### Acquire points and ordering + +Rule (from the design): acquire the budget **last**, immediately before +`GetSQE`, after every other blocking acquisition (FD open, req-pool alloc, +write-buffer alloc). This gives a consistent global order: FD/mutex → +pools/buffers → budget → SQE, and guarantees a task never holds budget while +blocked on another pool. + +To be precise about the two phases: *waiting inside* `Acquire` is expected — +the task parks on the budget's `WaitingZone` holding no budget (the counter +is only incremented on admission), and `PollComplete` wakes it regardless of +scheduling. The "no yield points" rule applies *after admission*: between +incrementing the counter and submitting the SQE there must be no voluntary +yield (`YieldToLowPQ`/`MaybeYield`) and no blocking on any resource whose +release could depend on this task. `GetSQE`'s ring-full wait is the one +tolerated block in that window: benign (slots are freed by kernel progress, +never by the blocked task) and unreachable in practice with caps ≪ +`io_queue_size`. + +| Entry point | Change | +|---|---| +| `IouringMgr::ReadPage` (`async_io_manager.cpp` ~484) | `read_budget_.Acquire(1)` at the top of the `read_page` lambda's retry loop, before `GetSQE`. Release is per-CQE, so each retry re-acquires. | +| `IouringMgr::ReadPages` (~565) | Acquire **per page** inside `send_req`, not per batch. Rationale: a 128-page compaction batch must not deadlock against a BG sub-budget of 64 (commit 2); per-page acquisition lets the task block mid-batch while already-submitted pages complete. This supersedes the "atomic batch acquire" idea discussed during design review. | +| `IouringMgr::WritePage` (~742) | `write_budget_.Acquire(1)` after `write_req_pool_->Alloc`, before `GetSQE`. | +| `IouringMgr::SubmitMergedWrite` (~777) | `write_budget_.Acquire(bytes / page_size)` after `merged_write_req_pool_->Alloc`, before `GetSQE`. | + +Exempt (unchanged): all metadata ops, manifest IO, `ReadFile` / +`WriteSnapshot` bulk paths, `Fdatasync` (instrumented only), +`ReadSegments` / `WriteSegments` (out of scope per design Non-Goals). + +### Release point + +In `PollComplete` (`async_io_manager.cpp` ~2010), per CQE, after the +existing per-type handling: `Release(cost)` on the matching budget. Wake +policy in commit 1 is a plain `WakeN` on the released budget's zone. + +### Options and validation + +- `kv_options.h`: add `max_inflight_read = 256`; redefine + `max_inflight_write` (page units, keep old default 32768 in this commit — + the default drops in commit 4). Add INI parsing in `kv_options.cpp` and + equality-operator entries. +- `eloq_store.cpp` option validation: `max_inflight_write != 0` already + enforced; nothing else hard-fails. `max_inflight_read == 0` disables the + read budget (documented). +- `WriteReqPool` stays sized by `max_inflight_write` (unchanged line in + `IouringMgr` ctor) — pool bound and QoS bound coincide by construction. + +### Stats + +`struct IoQosStats` (per budget: current, high-watermark, blocked count, +cumulative blocked µs) + `IouringMgr::GetIoQosStats()`. Wire into the +`ELOQSTORE_WITH_TXSERVICE` metrics meter behind the existing +`EnableMetrics()` guard; otherwise reachable from tests via the store's +shard accessors. Add an fdatasync counter + latency accumulator in +`SyncFiles`/`FdatasyncFiles` in the same commit (evaluation step 2 needs +it). + +### Shutdown / abort audit (do during code review of commit 1) + +- Tasks blocked in a budget `WaitingZone` are `TaskStatus::Blocked`, so + `TaskManager::NumActive() > 0` keeps `Shard::WorkLoop` spinning and + `PollComplete` continues releasing budget — no shutdown hang. Same + property as the existing `WriteReqPool::waiting_`. +- `AbortWrite` calls `WaitIo()` first; all of the aborting task's CQEs drain + through `PollComplete`, releasing their budget. No manual cleanup needed. +- Kill-point paths (`TEST_KILL_POINT_WEIGHT("WritePage", ...)`) fire before + budget acquisition — keep it that way. + +## Commit 2 — M2: FG/BG read sub-budget + +> **Status: implemented** (2026-07-02). As planned, with two refinements: +> (a) the sub-budget lives inside `IoBudget` (`SetBgCap` + `Acquire/Release` +> taking a `background` flag) rather than as a separate class, so the write +> budget reuses the same type with no sub-budget configured; per-class +> oversized-request escape and FIFO zones as described; stats gained a +> `bg_read_` slice and a BG-read inflight gauge. (b) The wake policy below +> ("wake FG waiters first") was found to starve background under sustained +> foreground saturation — recycling alone cannot even bootstrap BG's share +> from zero. Replaced by a demand-gated reservation: while BG waiters +> queue, their unused entitlement is off-limits to new FG admissions; +> release wakes BG first and forwards unused wake credits to FG +> (`WaitingZone::WakeN` now returns the count actually woken). + +- `KvTask::IsBackground()` in `tasks/task.h`: `BatchWrite`, + `BackgroundWrite`, `EvictFile`, `Prewarm` → true; explicitly not + `ReadOnly()`. +- Extend the read budget: `bg_inflight_` counter and a second `WaitingZone` + (`bg_waiting_`). Acquire for a BG task additionally checks + `bg_inflight_ + cost > bg_cap_`. Release decrements both counters when + the completing request's `task->IsBackground()`. +- Wake policy: on read-budget release, wake FG waiters first (`waiting_`), + then BG waiters only while `bg_inflight_ < bg_cap_` and total headroom + remains. Spurious wakes are safe (acquire re-checks in its while loop). +- Option `bg_read_ratio = 25` (percent, clamp 1–100); + `bg_cap_ = max_inflight_read * ratio / 100`, min 1 page... but see next + line: `bg_cap_` must be ≥ 1 and the per-page acquisition from commit 1 + guarantees any batch size makes progress through an arbitrarily small + sub-budget. + +## Commit 3 — benchmark and calibration tooling + +> **Status: implemented** (2026-07-02). Delivered as a standalone binary +> `benchmark/interference_bench.cpp` (+ `opts_interference.ini`) rather than +> extending `eloq_store_bm`, and `scripts/io_calibration_sweep.sh`. Two +> deviations from the sketch below: (a) the storm is a **rotating strided +> partial overwrite** (3-of-5 keys, ~3000B one-KV-per-page values, shifted +> each round) — a bulk overwrite leaves files fully dead and compaction +> generates no read traffic at all; (b) latency percentiles are exact +> (raw-sample sort at phase end), not sliding-window. The report includes +> per-shard IoQosStats deltas (BG-read watermark/blocked, fdatasync +> count/latency), giving the sweep direct visibility into whether the +> budgets engaged. Smoke-verified: storm lifts read p99 ~2.3× at toy scale, +> bg_read watermark pins at the configured sub-budget cap. + +- **Interference benchmark**: mixed scenario: steady point-read load at + fixed QD on N partitions while a write/compaction storm runs. Report read + p50/p99/p99.9 and BG write MB/s. Accept knob overrides via the existing + `opts_*.ini` mechanism so sweeps are scriptable. +- **fio calibration script** (`scripts/io_calibration_sweep.sh`): fixed + 4KB randread job + stepped sequential-write job against the target + device; emits the p99-vs-write-MB/s curve (design evaluation step 1). + Document drive preconditioning and SLC-exhaustion runtime in the script + header. +- Run design evaluation step 0 (re-baseline on top of #455) with this + benchmark before tightening any default. + +## Commit 4 — retire superseded throttles (gated on benchmarks) + +> **Status: implemented** (2026-07-03), gated on the WSL interference +> campaign (real-device confirmation still advisable before release). +> Details: `WritePage` throttle branch removed (budget admission comment +> left in its place); `max_write_batch_pages` deprecated — parsed with a +> LOG(WARNING), validation kept for compat, field documented as no-effect; +> `max_inflight_write` default 32768 → 512, validated by a {512, 2048, +> 32768} sweep (512 binds marginally — hwm pinned, 6–26 blocks/30s — at +> equal-or-best write throughput and read tails; natural demand ceiling was +> 2048 = 8 concurrent 1MB merged writes). One behavioral consequence: with +> enable_data_page_cache, write-promotion pins are now bounded by +> max_inflight_write instead of the per-task drain; the data_page_cache OOM +> test was recalibrated to `max_inflight_write = 1` (tiny 2-slot pool). +> Earlier same-day change under this commit's umbrella: +> max_inflight_read default 256 → 32, bg_read_ratio stays 25. + +Only after the interference benchmark confirms M1+M2 alone protect +foreground p99 (design evaluation step 4): + +- Remove the `inflight_io_ >= max_write_batch_pages → WaitWrite()` branch in + `WriteTask::WritePage` (`write_task.cpp` ~304). Keep the CPU yields and + the terminal `WaitWrite()`. +- Drop `max_inflight_write` default 32768 → calibrated value (~512). + Release-notes entry: behavioral change for deployments setting it + explicitly. +- Deprecate `max_write_batch_pages` in `kv_options.h` (keep parsing, + ignore, log a warning) for one release before deletion. + +## Commit 5 — M3 rate limiter (deferred) + +Sketch only; build if commit-4 benchmarks show sustained-compaction tail +degradation that inflight caps don't remove: token bucket (bytes) in +`IouringMgr`, refilled from the shard `WorkLoop` clock each round +(`ReadTimeMicroseconds` is already sampled per round), debited in +`WritePage`/`SubmitMergedWrite` when `task->IsBackground()`, bucket capacity +≤ 4 refill intervals, `bg_write_rate_limit = 0` disables. + +## Testing plan + +> **Status (2026-07-03):** unit-test items 1–6 all implemented in +> `tests/io_qos.cpp` (11 cases). Notes: item 4's "two concurrent merged +> writes" requires multiple partitions' write tasks — a single task's +> flushes don't overlap (it yields per page while filling the next buffer); +> item 5 uses the read-only-directory injection (kill points SIGTERM the +> process — they are crash-recovery tooling, not error injection); item 6 +> verifies Stop() drains tasks queued behind a 1-page budget. Regression: +> full suite run (memlock binaries via systemd-run on WSL); db_stress with +> tiny caps (read 4 / bg 1 / write 8, so append merged writes exercise the +> oversized-admission path every flush) run ≥100 iterations across +> non-append + append modes — db_stress needs `active_width <= max_key` +> or its sliding window indexes out of bounds (harness limitation, debug +> builds assert). ASAN: two modes exist — `WITH_ASAN=ON` (fully sound, +> including coroutine stacks) hard-requires a locally built +> `libboost_context-asan` (Boost.Context compiled with BOOST_USE_ASAN + +> ucontext; not a distro package); alternatively, injecting +> `-fsanitize=address` via CMAKE_CXX_FLAGS with WITH_ASAN=OFF links stock +> boost and gives full **heap** coverage (UAF/overflow/leaks — what the +> budget/pool plumbing needs) with degraded stack coverage inside coroutine +> frames. The QoS suites were run under the flag-injection mode on this +> box. +> +> Results (2026-07-03, WSL box): full suite 32/32 binaries green (memlock +> suites via systemd-run; persist's one failure was cross-load contention, +> 3/3 clean serially). db_stress: 120/120 iterations pass (60 non-append + +> 60 append, tiny caps). One real commit-4 fallout found by the sweep: +> data_page_cache "OOM retry under concurrent writes" became ~1-in-5 flaky +> — it runs append mode with a pool too small for a write-buffer pool, so +> writes take the non-append path where the removed per-task drain used to +> bound write-promotion pins; fixed by bounding pins via +> `max_inflight_write = 32` in the test (12/12 pass after). Same class of +> consequence as the other recalibrated OOM test; production pools are +> orders of magnitude above the ≤ max_inflight_write pin bound. + +### Unit tests (new `tests/io_qos.cpp`, Catch2, wired into `tests/CMakeLists.txt`) + +Use `tests/common.h` presets; add a `qos_opts` preset with deliberately tiny +caps to make blocking paths hot: + +1. **Accounting invariants**: mixed read/write workload on `default_opts` + + `append_opts` variants with `max_inflight_read = 4`, + `max_inflight_write = 8`; after quiesce, assert both budgets' current + == 0 and high-watermark ≤ cap. Repeat with caps disabled (0) — stats + stay zero, results identical. +2. **Oversized batch vs. tiny cap**: values with >128 overflow pages (forces + `GetOverflowValue`'s 128-page `ReadPages` batches) under + `max_inflight_read = 4` — completes, no deadlock (validates per-page + acquisition). +3. **BG sub-budget**: issue a `CompactRequest` concurrently with a stream of + point reads under small caps; assert BG high-watermark ≤ `bg_cap_`, + foreground reads complete, and compaction finishes (no starvation in + either direction). +4. **Merged-write cost**: append mode with `max_inflight_write = 256` and + 1MB write buffers — two merged writes cannot be in flight together + (watermark ≤ 256); with 512 they can. +5. **Error paths**: kill-point injection on `WritePage` (existing + `TEST_KILL_POINT_WEIGHT`) and a failing write (read-only dir trick or + fault injection as in `large_value_fault_injection.cpp`) — budget + returns to zero after `AbortWrite`. +6. **Shutdown while blocked**: stop the store while tasks are queued behind + a 1-page budget; store drains and stops cleanly. + +### Regression and stress + +- Full `ctest --test-dir build/tests/` with default options (must be + behavior-neutral before commit 4) and again with the tiny-cap `qos_opts` + exported via an env override if practical. +- `db_stress` run with tiny caps in both append and non-append modes; + repeat-run (≥100 iterations) to shake out wake/ordering races — budget + bugs manifest as rare hangs, same failure mode as the historical + free-list ABA issue, so favor iteration count over single-run length. +- ASAN build (`build-asan/`) for the new plumbing. +- On WSL2 dev machines, launch memlock-sensitive suites via + `systemd-run --user --pipe --wait --property=LimitMEMLOCK=2G` (ulimit is + unreliable there); do not shrink test parameters to dodge it. + +### Performance acceptance (per target device, after commit 3) + +- **Success criterion** (from evaluation step 0 re-baseline): read p99 + during compaction ≤ agreed multiple of idle p99 (set the number from the + re-baseline gap, not a priori). +- **No-regression guards**: pure-read throughput and pure-write throughput + within noise (±3%) of pre-QoS baseline at default caps — the hot-path + cost is two counter checks per IO, so any regression indicates a wake + storm or false blocking. +- Sweeps: `max_inflight_read` ∈ {64, 128, 256, 512}, `bg_read_ratio` ∈ + {10, 25, 50}, `max_inflight_write` ∈ {256, 512, 1024}, + `max_write_concurrency` ∈ {1, 2, 4, 8}; record read tails + BG + throughput; pick defaults at the knees. + +### Documentation updates (same commits as code) + +- `07-io-stack.md`: budgets, new UserDataTypes, acquire/release points. +- `04-execution-model.md`: budget blocking as a new task wait state, + interaction with `WaitingZone`/`MaybeYield`. +- `08-data-lifecycle.md`: compaction/GC now run under the BG read budget. +- `io_qos.md`: mark mechanisms as implemented as they land. diff --git a/include/async_io_manager.h b/include/async_io_manager.h index 19a00292e..6f16819a7 100644 --- a/include/async_io_manager.h +++ b/include/async_io_manager.h @@ -54,6 +54,112 @@ class ManifestFile virtual KvError SkipPadding(size_t n) = 0; }; +/** + * @brief Per-shard IO QoS statistics (see docs/design/io_qos.md). + * + * Snapshot semantics follow TailScratchAcquireCount: counters are mutated + * only by the owning shard thread and read without synchronization by + * tests/diagnostics; values are exact once the shard is quiesced. + */ +struct IoQosStats +{ + struct Budget + { + uint32_t inflight_{0}; // pages currently admitted + uint32_t high_watermark_{0}; // max pages ever admitted + uint64_t blocked_count_{0}; // acquisitions that had to wait + uint64_t blocked_us_{0}; // cumulative wait time + // Cumulative pages ever admitted. For the write budget this is the + // store-issued device write volume in pages (user data pages, + // compaction relocations, index pages — everything metered by the + // budget), i.e. the store-side counterpart of device write MB/s for + // comparing against the fio calibration curve. Excludes manifest + // appends, fdatasync, and segment IO (unbudgeted). + uint64_t admitted_pages_{0}; + }; + Budget read_; + // Background slice of read_ (M2): pages admitted by background tasks, + // bounded by the bg sub-budget. Always <= read_ component-wise. + Budget bg_read_; + Budget write_; + uint64_t fdatasync_count_{0}; // write-path fdatasync ops (FdatasyncFiles) + uint64_t fdatasync_us_{0}; // cumulative batch wall time +}; + +/** + * @brief Per-shard in-flight page-IO budget (M1/M2 in docs/design/io_qos.md). + * + * Counts admitted, not-yet-completed page IO in 4KB-page units. Tasks block + * in Acquire when admission would exceed the cap; IouringMgr::PollComplete + * releases per CQE and wakes waiters, so release never depends on the + * blocked task being scheduled. + * + * Optional background sub-budget (M2): when `bg_cap_` is non-zero, + * acquisitions with `background = true` are additionally bounded by + * `bg_inflight_ <= bg_cap_`. Background never exceeds its slice. Foreground + * may consume the entire budget while background has no queued demand; once + * background waiters exist, their unused entitlement (bg_cap_ - + * bg_inflight_) is reserved and new foreground admissions leave it alone, + * so background always ramps to its share — sustained foreground + * saturation cannot starve it. Each class waits on its own FIFO zone; + * release wakes background waiters first (freed units are reserved for + * them while they queue) and forwards unused wake credits to foreground. + * + * A cap of 0 disables the budget (Acquire/Release are no-ops). A request + * whose cost exceeds the (sub-)cap (e.g. a merged write larger than a small + * configured cap) is admitted alone once the relevant count drains to zero, + * so in-flight IO is bounded by max(cap, single-request cost) and progress + * is guaranteed. + */ +class IoBudget +{ +public: + void SetCap(uint32_t cap) + { + cap_ = cap; + } + void SetBgCap(uint32_t bg_cap) + { + bg_cap_ = bg_cap; + } + void Acquire(uint32_t cost, bool background = false); + void Release(uint32_t cost, bool background = false); + IoQosStats::Budget Stats() const + { + return {inflight_, + high_watermark_, + blocked_count_, + blocked_us_, + admitted_pages_}; + } + IoQosStats::Budget BgStats() const + { + return {bg_inflight_, + bg_high_watermark_, + bg_blocked_count_, + bg_blocked_us_, + bg_admitted_pages_}; + } + +private: + uint32_t cap_{0}; + uint32_t inflight_{0}; + uint32_t bg_cap_{0}; // 0 = no background sub-budget + uint32_t bg_inflight_{0}; + // The remaining fields are observability only (tests, tuning, metrics); + // admission decisions read nothing but the caps and inflight counters. + uint32_t high_watermark_{0}; + uint64_t blocked_count_{0}; + uint64_t blocked_us_{0}; + uint64_t admitted_pages_{0}; + uint32_t bg_high_watermark_{0}; + uint64_t bg_blocked_count_{0}; + uint64_t bg_blocked_us_{0}; + uint64_t bg_admitted_pages_{0}; + WaitingZone waiting_; // foreground waiters + WaitingZone bg_waiting_; // background waiters +}; + using ManifestFilePtr = std::unique_ptr; // TODO(zhanghao): consider using inheritance instead of variant @@ -196,6 +302,16 @@ class AsyncIoManager return 0; } + /** + * @brief Per-shard IO QoS statistics (in-flight page-IO budgets and + * fdatasync accounting). Default zeros for managers without budgets + * (MemStoreMgr). See docs/design/io_qos.md. + */ + virtual IoQosStats GetIoQosStats() const + { + return {}; + } + /** * @brief Read K segments into pre-registered buffers. * @@ -768,7 +884,15 @@ class IouringMgr : public AsyncIoManager KvTask, BaseReq, WriteReq, - MergedWriteReq + MergedWriteReq, + // Data-page reads charged against the read IO budget (see + // docs/design/io_qos.md). Named + PageRead: the + // payload and completion handling are identical to the plain + // KvTask / BaseReq types, plus a read-budget release. The distinct + // types exist so PollComplete can tell budgeted data-page reads + // apart from metadata ops that share the plain payload types. + KvTaskPageRead, // ReadPage (single page), payload = KvTask* + BaseReqPageRead // ReadPages (one page of a batch), payload = BaseReq* }; struct BaseReq @@ -1032,6 +1156,15 @@ class IouringMgr : public AsyncIoManager WaitingZone waiting_sqe_; uint32_t prepared_sqe_{0}; + // Per-shard in-flight page-IO budgets (M1, docs/design/io_qos.md). + // Reads and writes are separate device resources with independent caps + // (max_inflight_read / max_inflight_write). + IoBudget read_budget_; + IoBudget write_budget_; + // Write-path fdatasync instrumentation (FdatasyncFiles batches). + uint64_t fdatasync_count_{0}; + uint64_t fdatasync_us_{0}; + // Counter for consecutive Submit() calls that skipped the kernel // entry (no prepared SQEs and IORING_SQ_TASKRUN not set). When the // ring is configured with IORING_SETUP_DEFER_TASKRUN, the kernel @@ -1135,6 +1268,28 @@ class IouringMgr : public AsyncIoManager return tail_scratch_acquire_count_; } + IoQosStats GetIoQosStats() const override + { + IoQosStats stats; + stats.read_ = read_budget_.Stats(); + stats.bg_read_ = read_budget_.BgStats(); + stats.write_ = write_budget_.Stats(); + stats.fdatasync_count_ = fdatasync_count_; + stats.fdatasync_us_ = fdatasync_us_; + return stats; + } + + /** + * @brief Write-budget cost of a merged write in 4KB-page units. + * Acquire (SubmitMergedWrite) and release (PollComplete) must use this + * same formula so the budget balances exactly. + */ + uint32_t MergedWriteCost(size_t bytes) const + { + const uint32_t page_size = options_->data_page_size; + return static_cast((bytes + page_size - 1) / page_size); + } + /** * @brief Pure-function form of BufIndexForAddress: searches @p chunks for * the one containing @p ptr and returns base_index + chunk_index, or diff --git a/include/eloq_store.h b/include/eloq_store.h index d09bce2bc..bed220d94 100644 --- a/include/eloq_store.h +++ b/include/eloq_store.h @@ -34,6 +34,7 @@ namespace eloqstore { class Shard; class EloqStore; +struct IoQosStats; enum class RequestType : uint8_t { @@ -979,6 +980,15 @@ class EloqStore */ size_t TailScratchAcquireCount(size_t shard_id) const; + /** + * @brief Per-shard IO QoS statistics (in-flight page-IO budgets, + * fdatasync accounting; see docs/design/io_qos.md). Counters are + * mutated by the shard thread and read here without synchronization — + * exact once the shard is quiesced, diagnostic-quality otherwise. + * Returns zeros for invalid shard IDs or managers without budgets. + */ + IoQosStats GetIoQosStats(size_t shard_id) const; + bool ExecAsyn(KvRequest *req); void ExecSync(KvRequest *req); diff --git a/include/eloqstore_metrics.h b/include/eloqstore_metrics.h index b19de54aa..f39e96c3f 100644 --- a/include/eloqstore_metrics.h +++ b/include/eloqstore_metrics.h @@ -47,6 +47,12 @@ inline const Name NAME_ELOQSTORE_OPEN_FILE_LIMIT{"eloqstore_open_file_limit"}; inline const Name NAME_ELOQSTORE_LOCAL_SPACE_USED{"eloqstore_local_space_used"}; inline const Name NAME_ELOQSTORE_LOCAL_SPACE_LIMIT{ "eloqstore_local_space_limit"}; +inline const Name NAME_ELOQSTORE_INFLIGHT_READ_PAGES{ + "eloqstore_inflight_read_pages"}; +inline const Name NAME_ELOQSTORE_INFLIGHT_BG_READ_PAGES{ + "eloqstore_inflight_bg_read_pages"}; +inline const Name NAME_ELOQSTORE_INFLIGHT_WRITE_PAGES{ + "eloqstore_inflight_write_pages"}; // Collection interval for Phase 9-11 gauge metrics (index buffer pool, page // pool, open file, local space) These metrics are collected every N diff --git a/include/kv_options.h b/include/kv_options.h index e71d28636..6fd293be6 100644 --- a/include/kv_options.h +++ b/include/kv_options.h @@ -73,12 +73,49 @@ struct KvOptions */ uint32_t io_queue_size = 4096; /** - * @brief Max amount of inflight write IO per shard. - * Only take effect in non-append write mode. + * @brief Per-shard cap on in-flight page-write IO, in 4KB-page units + * (docs/design/io_qos.md M1). A merged append-mode write of N bytes + * counts as N / data_page_size pages, so the cap means the same thing + * in append and non-append mode. Also sizes the write request pools. + * Cannot be zero. + * + * NOTE: before the IO QoS work this option only sized the non-append + * write request pool and defaulted to 32768 (effectively unbounded); + * deployments that set it explicitly should re-derive their value as a + * queue-depth cap. */ - uint32_t max_inflight_write = 32 << 10; + uint32_t max_inflight_write = 512; /** - * @brief The maximum number of pages per batch for the write task. + * @brief Per-shard cap on in-flight page-read IO, in 4KB-page units + * (docs/design/io_qos.md M1). Applies to data-page reads + * (ReadPage/ReadPages); metadata and segment IO are exempt. + * 0 disables the read budget. + * + * This is the device-calibration knob of the QoS sizing contract (see + * "Sizing contract" in docs/design/io_qos.md): size it near the + * device's bandwidth-delay product, c * max_random_read_IOPS * + * unloaded_read_latency / num_threads with c ~ 2-4. Foreground + * `read_blocked` staying ~0 under representative load validates the + * value; nonzero means undersized. + */ + uint32_t max_inflight_read = 32; + /** + * @brief Background share of max_inflight_read, in percent (clamped to + * 1..100; docs/design/io_qos.md M2). Page reads issued by background + * tasks (batch write, compaction, GC, prewarm) are bounded by this + * sub-budget so they cannot crowd out foreground point reads. + * Foreground reads may use the entire read budget. No effect when the + * read budget is disabled (max_inflight_read = 0). + */ + uint32_t bg_read_ratio = 25; + /** + * @brief DEPRECATED — no effect. Formerly the per-write-task in-flight + * page cap (the task drained to zero via WaitWrite once it had this + * many writes outstanding). Superseded by the shard-wide write budget + * `max_inflight_write` (docs/design/io_qos.md, plan commit 4), which + * bounds in-flight write pages across all tasks without the + * drain-to-zero sawtooth. Parsed and validated for compatibility; + * scheduled for removal one release after deprecation. */ uint32_t max_write_batch_pages = 32; /** diff --git a/include/storage/shard.h b/include/storage/shard.h index fb3740093..521fb5a79 100644 --- a/include/storage/shard.h +++ b/include/storage/shard.h @@ -78,6 +78,13 @@ class Shard return DurationMicroseconds(cur_resume_start_us_); } + // Cheap TSC-based clock (rdtsc / calibrated cycles-per-us; ARM virtual + // counter on aarch64). Public so shard-thread code outside Shard (e.g. + // IoBudget blocked-time accounting) can time intervals without a + // clock_gettime call. + uint64_t ReadTimeMicroseconds(); + uint64_t DurationMicroseconds(uint64_t start_us); + std::atomic io_mgr_and_page_pool_inited_{false}; #ifdef ELOQ_MODULE_ENABLED @@ -144,10 +151,6 @@ class Shard // module worker (WorkOneRound), the sole consumer of requests_. void DrainPendingRequests(); - uint64_t ReadTimeMicroseconds(); - - uint64_t DurationMicroseconds(uint64_t start_us); - template void StartTask(KvTask *task, KvRequest *req, F lbd) { diff --git a/include/tasks/task.h b/include/tasks/task.h index e80f7c7c2..5f063ef79 100644 --- a/include/tasks/task.h +++ b/include/tasks/task.h @@ -194,6 +194,26 @@ class KvTask { return Type() < TaskType::BatchWrite; } + /** + * @brief Background tasks' page reads are charged against the background + * sub-budget of the read IO budget (docs/design/io_qos.md M2), so they + * cannot crowd out foreground point reads at the device. Deliberately + * not derived from ReadOnly(): EvictFile and Prewarm are read-only but + * background. + */ + bool IsBackground() const + { + switch (Type()) + { + case TaskType::BatchWrite: + case TaskType::BackgroundWrite: + case TaskType::EvictFile: + case TaskType::Prewarm: + return true; + default: + return false; + } + } void Yield(); void YieldToLowPQ(); /** @@ -224,7 +244,13 @@ class WaitingZone WaitingZone() = default; void Wait(KvTask *task); void WakeOne(); - void WakeN(size_t n); + /** + * @brief Wake up to n waiters in FIFO order. + * @return The number actually woken (< n when the zone drains), so the + * caller can forward unused wake credits to another zone (see + * IoBudget::Release). + */ + size_t WakeN(size_t n); void WakeAll(); bool Empty() const; diff --git a/scripts/io_calibration_sweep.sh b/scripts/io_calibration_sweep.sh new file mode 100755 index 000000000..358d9fc23 --- /dev/null +++ b/scripts/io_calibration_sweep.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# +# IO QoS device calibration sweep (docs/design/io_qos.md, evaluation step 1). +# +# Runs a fixed 4KB random-read fio job while stepping a rate-limited +# sequential-write job through increasing MB/s targets, and emits a CSV of +# write_target_MBps, write_actual_MBps, read_iops, read_p50_us, +# read_p99_us, read_p999_us +# The knee of the p99-vs-write-rate curve is the per-device background write +# budget; divide by num_threads for the per-shard bg budget / M3 rate limit. +# +# IMPORTANT — read before trusting the numbers: +# * PRECONDITION the drive: a fresh/trimmed SSD serves reads from empty FTL +# mappings and absorbs writes into a pristine SLC cache, both of which +# flatter the results. Fill the test file region at least once (the script +# lays out the file with a full sequential write pass unless it already +# exists) and ideally run `--precondition` (two full overwrite passes) +# on a drive that has seen real use. +# * SLC-CACHE EXHAUSTION: consumer drives fold SLC->TLC once the cache +# fills; read tails get much worse and unstable after that point. Size +# per-step runtime (--step-secs, default 60) and the write rates so the +# total bytes written per step exceed the SLC cache if you want +# steady-state numbers; watch for a step whose actual write MB/s sags +# below the target — that is the fold-over signature. +# * Run against a FILE on the target filesystem (not the raw device) to +# include filesystem effects, or a raw block device for pure-device +# numbers. Never point this at a device with data you care about when +# using --device. +# +# Requires: fio, python3. +# +# Usage: +# scripts/io_calibration_sweep.sh --dir /mnt/nvme/fio-test \ +# [--size 32G] [--step-secs 60] [--read-qd 32] \ +# [--rates "0 50 100 200 400 800 1600"] [--precondition] + +set -euo pipefail + +DIR="" +SIZE="32G" +STEP_SECS=60 +READ_QD=32 +RATES="0 50 100 200 400 800 1600" +PRECONDITION=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --dir) DIR="$2"; shift 2 ;; + --size) SIZE="$2"; shift 2 ;; + --step-secs) STEP_SECS="$2"; shift 2 ;; + --read-qd) READ_QD="$2"; shift 2 ;; + --rates) RATES="$2"; shift 2 ;; + --precondition) PRECONDITION=1; shift ;; + *) echo "unknown arg: $1" >&2; exit 1 ;; + esac +done + +[[ -n "$DIR" ]] || { echo "usage: $0 --dir [...]" >&2; exit 1; } +command -v fio >/dev/null || { echo "fio not installed" >&2; exit 1; } +command -v python3 >/dev/null || { echo "python3 not installed" >&2; exit 1; } + +mkdir -p "$DIR" +READ_FILE="$DIR/calib_read.bin" +WRITE_FILE="$DIR/calib_write.bin" + +layout() { + local file="$1" passes="$2" + for ((i = 0; i < passes; i++)); do + echo "# layout pass $((i + 1))/$passes of $file" >&2 + fio --name=layout --filename="$file" --size="$SIZE" --rw=write \ + --bs=1M --iodepth=8 --direct=1 --ioengine=libaio \ + --output-format=terse >/dev/null + done +} + +# Lay out both files so reads never hit holes; precondition = extra passes. +PASSES=$((PRECONDITION ? 2 : 1)) +[[ -f "$READ_FILE" && $PRECONDITION -eq 0 ]] || layout "$READ_FILE" "$PASSES" +[[ -f "$WRITE_FILE" && $PRECONDITION -eq 0 ]] || layout "$WRITE_FILE" "$PASSES" + +echo "write_target_MBps,write_actual_MBps,read_iops,read_p50_us,read_p99_us,read_p999_us" + +for rate in $RATES; do + OUT=$(mktemp) + if [[ "$rate" == "0" ]]; then + # Baseline: reads only. + fio --output-format=json --output="$OUT" \ + --name=randread --filename="$READ_FILE" --size="$SIZE" \ + --rw=randread --bs=4k --iodepth="$READ_QD" --direct=1 \ + --ioengine=libaio --time_based --runtime="$STEP_SECS" \ + >/dev/null + else + fio --output-format=json --output="$OUT" \ + --name=randread --filename="$READ_FILE" --size="$SIZE" \ + --rw=randread --bs=4k --iodepth="$READ_QD" --direct=1 \ + --ioengine=libaio --time_based --runtime="$STEP_SECS" \ + --name=seqwrite --filename="$WRITE_FILE" --size="$SIZE" \ + --rw=write --bs=1M --iodepth=4 --direct=1 \ + --ioengine=libaio --time_based --runtime="$STEP_SECS" \ + --rate=,"${rate}m" \ + >/dev/null + fi + python3 - "$OUT" "$rate" <<'EOF' +import json +import sys + +with open(sys.argv[1]) as f: + data = json.load(f) +rate = sys.argv[2] +read_iops = 0.0 +read_p = {"50.000000": 0, "99.000000": 0, "99.900000": 0} +write_mbps = 0.0 +for job in data["jobs"]: + if job["jobname"] == "randread": + r = job["read"] + read_iops = r["iops"] + # clat_ns percentiles -> us + pct = r.get("clat_ns", {}).get("percentile", {}) + for k in read_p: + read_p[k] = pct.get(k, 0) / 1000.0 + elif job["jobname"] == "seqwrite": + write_mbps = job["write"]["bw_bytes"] / (1 << 20) +print( + f"{rate},{write_mbps:.0f},{read_iops:.0f}," + f"{read_p['50.000000']:.0f},{read_p['99.000000']:.0f}," + f"{read_p['99.900000']:.0f}" +) +EOF + rm -f "$OUT" +done diff --git a/src/async_io_manager.cpp b/src/async_io_manager.cpp index 8e676ab9d..199da7c99 100644 --- a/src/async_io_manager.cpp +++ b/src/async_io_manager.cpp @@ -166,6 +166,99 @@ bool AsyncIoManager::IsIdle() return true; } +void IoBudget::Acquire(uint32_t cost, bool background) +{ + if (cap_ == 0) + { + return; + } + const bool use_bg = background && bg_cap_ != 0; + // Admission conditions. `inflight != 0` implements the oversized-request + // escape per class: a request with cost > (sub-)cap is admitted alone + // once the relevant count drains, guaranteeing progress (see IoBudget + // doc comment). Background is additionally bounded by its sub-budget. + // Foreground must not take units reserved for *queued* background + // demand (bg_cap_ - bg_inflight_ while bg_waiting_ is non-empty): + // without that reservation, sustained foreground saturation would + // starve background forever — every freed unit would be re-acquired by + // a foreground waiter and compaction could never bootstrap its share. + // While background has no queued demand, foreground may use the entire + // budget. + auto must_wait = [this, cost, use_bg]() + { + if (use_bg) + { + if (inflight_ + cost > cap_ && inflight_ != 0) + { + return true; + } + return bg_inflight_ + cost > bg_cap_ && bg_inflight_ != 0; + } + const uint32_t reserved = + (bg_cap_ != 0 && !bg_waiting_.Empty()) ? bg_cap_ - bg_inflight_ : 0; + return inflight_ + cost > cap_ - reserved && inflight_ != 0; + }; + // Each class queues behind its own existing waiters (approximate FIFO + // across arrivals within the class). + WaitingZone &zone = use_bg ? bg_waiting_ : waiting_; + if (!zone.Empty() || must_wait()) + { + // TSC-based clock (see Shard::ReadTimeMicroseconds): Acquire always + // runs on the shard thread, after Shard::Init calibrated the TSC. + const uint64_t start_us = shard->ReadTimeMicroseconds(); + (use_bg ? bg_blocked_count_ : blocked_count_)++; + do + { + zone.Wait(ThdTask()); + } while (must_wait()); + const uint64_t waited_us = shard->DurationMicroseconds(start_us); + (use_bg ? bg_blocked_us_ : blocked_us_) += waited_us; + } + inflight_ += cost; + admitted_pages_ += cost; + if (inflight_ > high_watermark_) + { + high_watermark_ = inflight_; + } + if (use_bg) + { + bg_inflight_ += cost; + bg_admitted_pages_ += cost; + if (bg_inflight_ > bg_high_watermark_) + { + bg_high_watermark_ = bg_inflight_; + } + } +} + +void IoBudget::Release(uint32_t cost, bool background) +{ + if (cap_ == 0) + { + return; + } + assert(inflight_ >= cost); + inflight_ -= cost; + if (background && bg_cap_ != 0) + { + assert(bg_inflight_ >= cost); + bg_inflight_ -= cost; + } + // Each freed page-unit can admit at most one waiter; over-waking is safe + // because woken tasks re-check the admission condition and re-wait. + // Background waiters are woken first: while background has queued + // demand and sub-budget room, freed units are reserved for it (see the + // admission rule in Acquire), so waking foreground for those units + // would be futile. Unused wake credits are forwarded to foreground — + // when background is saturated or idle, all credits go to foreground. + size_t woken = 0; + if (bg_cap_ != 0 && bg_inflight_ < bg_cap_) + { + woken = bg_waiting_.WakeN(cost); + } + waiting_.WakeN(cost - woken); +} + IouringMgr::IouringMgr(const KvOptions *opts, uint32_t fd_limit) : AsyncIoManager(opts), fd_limit_(fd_limit) { @@ -176,6 +269,24 @@ IouringMgr::IouringMgr(const KvOptions *opts, uint32_t fd_limit) uint32_t pool_size = options_->max_inflight_write; write_req_pool_ = std::make_unique(pool_size); merged_write_req_pool_ = std::make_unique(pool_size); + + // In-flight page-IO budgets (docs/design/io_qos.md M1/M2). The write cap + // shares max_inflight_write with the request-pool sizing above, so the + // pool bound and the QoS bound coincide by construction. The read budget + // carries the background sub-budget; the write budget has none — all + // page writes come from write tasks, which are background by definition. + read_budget_.SetCap(options_->max_inflight_read); + write_budget_.SetCap(options_->max_inflight_write); + if (options_->max_inflight_read != 0) + { + const uint32_t ratio = + std::clamp(options_->bg_read_ratio, 1, 100); + const uint32_t bg_cap = std::max( + 1, + static_cast(uint64_t{options_->max_inflight_read} * + ratio / 100)); + read_budget_.SetBgCap(bg_cap); + } } IouringMgr::~IouringMgr() @@ -739,7 +850,12 @@ std::pair IouringMgr::ReadPage(const TableIdent &tbl_id, int res; do { - io_uring_sqe *sqe = GetSQE(UserDataType::KvTask, ThdTask()); + // Read-budget admission (io_qos.md M1/M2): acquired last, right + // before SQE prep; released per CQE in PollComplete, so each + // retry iteration re-acquires. Background tasks are additionally + // bounded by the BG sub-budget. + read_budget_.Acquire(1, ThdTask()->IsBackground()); + io_uring_sqe *sqe = GetSQE(UserDataType::KvTaskPageRead, ThdTask()); if (fd.second) { sqe->flags |= IOSQE_FIXED_FILE; @@ -838,8 +954,12 @@ KvError IouringMgr::ReadPages(const TableIdent &tbl_id, auto send_req = [this](ReadReq *req) { + // Read-budget admission (io_qos.md M1/M2) is per page, not per batch: + // the task may block mid-batch while already-submitted pages + // complete, so a batch larger than the (sub-)budget cannot deadlock. + read_budget_.Acquire(1, req->task_->IsBackground()); auto [fd, registered] = req->fd_ref_.FdPair(); - io_uring_sqe *sqe = GetSQE(UserDataType::BaseReq, req); + io_uring_sqe *sqe = GetSQE(UserDataType::BaseReqPageRead, req); if (registered) { sqe->flags |= IOSQE_FIXED_FILE; @@ -983,6 +1103,9 @@ KvError IouringMgr::WritePage(const TableIdent &tbl_id, auto [fd, registered] = fd_ref.FdPair(); WriteReq *req = write_req_pool_->Alloc(std::move(fd_ref), std::move(page)); + // Write-budget admission (io_qos.md M1): after every other blocking + // acquisition (FD, req pool), immediately before SQE prep. + write_budget_.Acquire(1); io_uring_sqe *sqe = GetSQE(UserDataType::WriteReq, req); if (registered) { @@ -1288,6 +1411,10 @@ KvError IouringMgr::SubmitMergedWrite(const TableIdent &tbl_id, static_cast(req->pages_.size() - 1); } + // Write-budget admission (io_qos.md M1): cost in 4KB-page units so the + // cap means the same thing in append and non-append mode. Must mirror + // the release cost computed from bytes_ in PollComplete. + write_budget_.Acquire(MergedWriteCost(bytes)); io_uring_sqe *sqe = GetSQE(UserDataType::MergedWriteReq, req); auto [fd, registered] = req->fd_ref_.FdPair(); if (registered) @@ -2019,14 +2146,24 @@ void IouringMgr::PollComplete() KvTask *task = nullptr; switch (type) { + case UserDataType::KvTaskPageRead: case UserDataType::KvTask: task = static_cast(ptr); + if (type == UserDataType::KvTaskPageRead) + { + read_budget_.Release(1, task->IsBackground()); + } task->io_res_ = cqe->res; task->io_flags_ = cqe->flags; break; + case UserDataType::BaseReqPageRead: case UserDataType::BaseReq: { BaseReq *req = static_cast(ptr); + if (type == UserDataType::BaseReqPageRead) + { + read_budget_.Release(1, req->task_->IsBackground()); + } req->res_ = cqe->res; req->flags_ = cqe->flags; task = req->task_; @@ -2052,6 +2189,12 @@ void IouringMgr::PollComplete() req->task_->WritePageCallback(std::move(req->page_), err); task = req->task_; write_req_pool_->Free(req); + // No class argument: the write budget has no background + // sub-budget (all page writes come from write tasks, i.e. + // background — see io_qos.md M2), so acquire and release both + // use the default. Must stay symmetric with WritePage's + // Acquire. + write_budget_.Release(1); break; } case UserDataType::MergedWriteReq: @@ -2085,6 +2228,10 @@ void IouringMgr::PollComplete() req->release_indices_[i]); } } + // No class argument (see the WriteReq case): the write budget + // has no background sub-budget. Cost must mirror + // SubmitMergedWrite's Acquire exactly. + write_budget_.Release(MergedWriteCost(req->bytes_)); merged_write_req_pool_->Free(req); continue; } @@ -2276,6 +2423,9 @@ KvError IouringMgr::FdatasyncFiles(const TableIdent &tbl_id, // Fsync all dirty files/directory. std::vector reqs; reqs.reserve(fds.size()); + // Instrumented for IO QoS evaluation (io_qos.md): fsync stalls are a + // distinct interference channel from page-IO queueing. + const uint64_t fsync_start_us = shard->ReadTimeMicroseconds(); for (LruFD::Ref &fd_ref : fds) { // FsyncReq elements have pointer stability, because we have reserved @@ -2290,6 +2440,8 @@ KvError IouringMgr::FdatasyncFiles(const TableIdent &tbl_id, io_uring_prep_fsync(sqe, fd, IORING_FSYNC_DATASYNC); } ThdTask()->WaitIo(); + fdatasync_count_ += reqs.size(); + fdatasync_us_ += shard->DurationMicroseconds(fsync_start_us); // Check results. KvError err = KvError::NoError; diff --git a/src/eloq_store.cpp b/src/eloq_store.cpp index 5f2fbe15b..d11d3401f 100644 --- a/src/eloq_store.cpp +++ b/src/eloq_store.cpp @@ -2451,6 +2451,16 @@ size_t EloqStore::TailScratchAcquireCount(size_t shard_id) const return io_mgr == nullptr ? 0 : io_mgr->TailScratchAcquireCount(); } +IoQosStats EloqStore::GetIoQosStats(size_t shard_id) const +{ + if (shard_id >= shards_.size() || shards_[shard_id] == nullptr) + { + return {}; + } + AsyncIoManager *io_mgr = shards_[shard_id]->IoManager(); + return io_mgr == nullptr ? IoQosStats{} : io_mgr->GetIoQosStats(); +} + bool EloqStore::IsStopped() const { return status_.load(std::memory_order_acquire) == Status::Stopped; diff --git a/src/kv_options.cpp b/src/kv_options.cpp index 96b73b2b4..944654eae 100644 --- a/src/kv_options.cpp +++ b/src/kv_options.cpp @@ -154,10 +154,23 @@ int KvOptions::LoadFromIni(const char *path) max_inflight_write = reader.GetUnsigned(sec_run, "max_inflight_write", 4096); } + if (reader.HasValue(sec_run, "max_inflight_read")) + { + max_inflight_read = + reader.GetUnsigned(sec_run, "max_inflight_read", 32); + } + if (reader.HasValue(sec_run, "bg_read_ratio")) + { + bg_read_ratio = reader.GetUnsigned(sec_run, "bg_read_ratio", 25); + } if (reader.HasValue(sec_run, "max_write_batch_pages")) { max_write_batch_pages = reader.GetUnsigned(sec_run, "max_write_batch_pages", 64); + LOG(WARNING) + << "Option max_write_batch_pages is deprecated and has no " + "effect; in-flight write IO is bounded by max_inflight_write " + "(see docs/design/io_qos.md)"; } if (reader.HasValue(sec_run, "coroutine_stack_size")) { @@ -398,6 +411,8 @@ bool KvOptions::operator==(const KvOptions &other) const manifest_limit == other.manifest_limit && fd_limit == other.fd_limit && io_queue_size == other.io_queue_size && max_inflight_write == other.max_inflight_write && + max_inflight_read == other.max_inflight_read && + bg_read_ratio == other.bg_read_ratio && max_write_batch_pages == other.max_write_batch_pages && coroutine_stack_size == other.coroutine_stack_size && num_retained_archives == other.num_retained_archives && diff --git a/src/storage/shard.cpp b/src/storage/shard.cpp index ed01eae8a..266edb6c1 100644 --- a/src/storage/shard.cpp +++ b/src/storage/shard.cpp @@ -1357,6 +1357,15 @@ void Shard::WorkOneRound() size_t local_space_used = io_mgr_->GetLocalSpaceUsed(); meter->Collect(metrics::NAME_ELOQSTORE_LOCAL_SPACE_USED, static_cast(local_space_used)); + + // Collect in-flight page-IO budget usage (io_qos.md M1) + IoQosStats qos = io_mgr_->GetIoQosStats(); + meter->Collect(metrics::NAME_ELOQSTORE_INFLIGHT_READ_PAGES, + static_cast(qos.read_.inflight_)); + meter->Collect(metrics::NAME_ELOQSTORE_INFLIGHT_BG_READ_PAGES, + static_cast(qos.bg_read_.inflight_)); + meter->Collect(metrics::NAME_ELOQSTORE_INFLIGHT_WRITE_PAGES, + static_cast(qos.write_.inflight_)); } } #endif diff --git a/src/tasks/task.cpp b/src/tasks/task.cpp index 1a33cfd8f..682d66440 100644 --- a/src/tasks/task.cpp +++ b/src/tasks/task.cpp @@ -505,9 +505,10 @@ void WaitingZone::WakeOne() } } -void WaitingZone::WakeN(size_t n) +size_t WaitingZone::WakeN(size_t n) { - for (size_t i = 0; i < n; i++) + size_t woken = 0; + while (woken < n) { KvTask *task = PopFront(); if (task == nullptr) @@ -516,7 +517,9 @@ void WaitingZone::WakeN(size_t n) } assert(task->status_ == TaskStatus::Blocked); task->Resume(); + woken++; } + return woken; } void WaitingZone::WakeAll() diff --git a/src/tasks/write_task.cpp b/src/tasks/write_task.cpp index 35e92e13a..1ad701249 100644 --- a/src/tasks/write_task.cpp +++ b/src/tasks/write_task.cpp @@ -302,16 +302,15 @@ KvError WriteTask::WritePage(VarPage page, FilePageId file_page_id) KvError err = IoMgr()->WritePage(tbl_ident_, std::move(page), file_page_id); CHECK_KV_ERR(err); - if (inflight_io_ >= opts->max_write_batch_pages) - { - // Avoid long running WriteTask block ReadTask/ScanTask - err = WaitWrite(); - CHECK_KV_ERR(err); - } - else - { - YieldToLowPQ(); - } + // In-flight write IO is bounded by the shard's write budget + // (max_inflight_write, enforced inside IoMgr()->WritePage; see + // docs/design/io_qos.md M1), which replaced the per-task + // max_write_batch_pages drain-to-zero throttle here: budget admission + // keeps a steady in-flight level instead of a sawtooth, and unlike the + // per-task cap it also counts compaction and concurrent write tasks. + // Completion errors are collected by the terminal WaitWrite() before + // UpdateMeta/SyncData. The yield below is CPU-side cooperation only. + YieldToLowPQ(); return KvError::NoError; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7262e7e93..5d42a39da 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -45,6 +45,7 @@ set(UTEST_SOURCES iouring_tail_scratch.cpp segment_allocator.cpp data_page_cache.cpp + io_qos.cpp ) string( REPLACE ".cpp" "" BASENAMES_UTEST "${UTEST_SOURCES}" ) diff --git a/tests/batch_write.cpp b/tests/batch_write.cpp index e6981387c..7c2c0d352 100644 --- a/tests/batch_write.cpp +++ b/tests/batch_write.cpp @@ -129,7 +129,6 @@ TEST_CASE("batch write abort releases pinned index pages", opts.num_threads = 1; opts.data_page_size = 4096; opts.buffer_pool_size = 4096; // Allow only a single MemCachedPage. - opts.max_write_batch_pages = 4; opts.auto_oom_retry_times = 0; auto build_entries = @@ -190,7 +189,6 @@ TEST_CASE("batch write task pool handles many partitions concurrently", opts.store_path = {test_path}; opts.num_threads = 1; // single shard, many partitions opts.buffer_pool_size = 4096 * 400; // enough for many pages - opts.max_write_batch_pages = 8; auto make_entries = [](uint32_t base, uint32_t count) { diff --git a/tests/cloud.cpp b/tests/cloud.cpp index 05ef8f196..62da7c015 100644 --- a/tests/cloud.cpp +++ b/tests/cloud.cpp @@ -447,12 +447,10 @@ TEST_CASE("cloud reuse cache enforces budgets across restarts", CleanupStore(options); - auto store = std::make_unique(options); - REQUIRE(store->Start(eloqstore::MainBranchName, 0) == - eloqstore::KvError::NoError); + eloqstore::EloqStore *store = InitStore(options, /*cleanup=*/false); eloqstore::TableIdent tbl_id{"reuse-cache", 0}; - MapVerifier writer(tbl_id, store.get()); + MapVerifier writer(tbl_id, store); writer.SetAutoClean(false); writer.SetAutoValidate(false); writer.SetValueSize(64 << 10); @@ -475,7 +473,7 @@ TEST_CASE("cloud reuse cache enforces budgets across restarts", store->Stop(); REQUIRE(store->Start(eloqstore::MainBranchName, 0) == eloqstore::KvError::NoError); - writer.SetStore(store.get()); + writer.SetStore(store); WriteBatches(writer, next_key, entries_per_batch, batches_per_phase); @@ -489,11 +487,11 @@ TEST_CASE("cloud reuse cache enforces budgets across restarts", store->Stop(); // Tighten the budget to 20MB and verify restore trims/existing files and - // future writes respect the new limit. + // future writes respect the new limit. InitStore (cleanup=false) replaces + // the previous instance while keeping the cached files on disk. options.local_space_limit = 20ULL << 20; - auto trimmed_store = std::make_unique(options); - REQUIRE(trimmed_store->Start("main", 0) == eloqstore::KvError::NoError); - writer.SetStore(trimmed_store.get()); + eloqstore::EloqStore *trimmed_store = InitStore(options, /*cleanup=*/false); + writer.SetStore(trimmed_store); WriteBatches(writer, next_key, entries_per_batch, batches_per_phase / 2); @@ -580,8 +578,7 @@ TEST_CASE("cloud startup restore removes empty idle partition directories", CleanupStore(options); - auto store = std::make_unique(options); - REQUIRE(store->Start() == eloqstore::KvError::NoError); + eloqstore::EloqStore *store = InitStore(options, /*cleanup=*/false); store->Stop(); const eloqstore::TableIdent tbl_id{"reuse_empty_idle", 0}; @@ -617,8 +614,7 @@ TEST_CASE("cloud startup restore removes partitions cleaned to empty", CleanupStore(options); - auto store = std::make_unique(options); - REQUIRE(store->Start() == eloqstore::KvError::NoError); + eloqstore::EloqStore *store = InitStore(options, /*cleanup=*/false); store->Stop(); const eloqstore::TableIdent tbl_id{"reuse_cleanup_idle", 0}; diff --git a/tests/common.cpp b/tests/common.cpp index d51c54eb7..c786283c3 100644 --- a/tests/common.cpp +++ b/tests/common.cpp @@ -8,13 +8,23 @@ #include "utils.h" -eloqstore::EloqStore *InitStore(const eloqstore::KvOptions &opts) +eloqstore::EloqStore *InitStore(const eloqstore::KvOptions &opts, bool cleanup) { static std::unique_ptr eloq_store = nullptr; // Tear down any prior store before constructing the new one, so the old // destructor's worker-thread joins and LRU-cached fd releases finish // before we count the new store's fd budget below. + // + // Every test-created store must go through InitStore — never construct + // an EloqStore directly in a test while another may be running. The + // process-global `eloq_store` pointer (Options()/Comp() plumbing) + // assumes at most one started store per process; a second concurrent + // instance leaves one store's teardown reading a nulled/foreign global + // (observed as a flaky SIGSEGV in Prewarmer::Shutdown). Tests that need + // to preserve on-disk/cloud state across store generations (warm + // restart, cache-trim) pass cleanup = false instead of bypassing + // InitStore. if (eloq_store) { if (!eloq_store->IsStopped()) @@ -23,11 +33,14 @@ eloqstore::EloqStore *InitStore(const eloqstore::KvOptions &opts) } eloq_store.reset(); } - if (!opts.cloud_store_path.empty()) + if (cleanup) { - S3TestClient s3_client(opts); + if (!opts.cloud_store_path.empty()) + { + S3TestClient s3_client(opts); + } + CleanupStore(opts); } - CleanupStore(opts); // EloqStore::Start() counts the *process-wide* `/proc/self/fd` and // subtracts it from `fd_limit`. When multiple test cases run in the diff --git a/tests/common.h b/tests/common.h index c5456787f..d158a38b6 100644 --- a/tests/common.h +++ b/tests/common.h @@ -78,7 +78,16 @@ const eloqstore::KvOptions cloud_archive_opts = { .pages_per_file_shift = 8, .data_append_mode = true, }; -eloqstore::EloqStore *InitStore(const eloqstore::KvOptions &opts); +/** + * Create (or replace) the per-binary test store. Stops and destroys any + * previously created store first, so at most one EloqStore instance is ever + * started in the process — required by the process-global Options() pointer. + * All test store creation must go through here; do not construct EloqStore + * directly in tests. Pass cleanup = false to keep existing local/cloud state + * (warm-restart and cache-reuse tests). + */ +eloqstore::EloqStore *InitStore(const eloqstore::KvOptions &opts, + bool cleanup = true); bool ValidateFileSizes(const eloqstore::KvOptions &opts); diff --git a/tests/data_page_cache.cpp b/tests/data_page_cache.cpp index 06905a97a..16f7a70d5 100644 --- a/tests/data_page_cache.cpp +++ b/tests/data_page_cache.cpp @@ -178,6 +178,13 @@ TEST_CASE("data page cache: OOM retry under concurrent writes", opts.store_path = {"/tmp/eloqstore"}; opts.num_threads = 1; opts.buffer_pool_size = 128 * 4096; // ~512 KB; comfortably fits one task + // The pool above is too small for a write-buffer pool, so these appends + // take the non-append WritePage path, where write-promotion pins on + // cached pages persist until the write IO completes. Bound the shard's + // in-flight writes so concurrent tasks' pins can't crowd the 128-slot + // pool past what 5 OOM retries can absorb (the deprecated per-task + // max_write_batch_pages drain used to provide this bound implicitly). + opts.max_inflight_write = 32; eloqstore::EloqStore *store = InitStore(opts); @@ -237,7 +244,11 @@ TEST_CASE("data page cache: OOM abort releases pinned pages", // Two slots: just enough for a small steady state, but the OOM batch // below intentionally exceeds it. opts.buffer_pool_size = 2 * 4096; - opts.max_write_batch_pages = 4; + // Keep at most one write in flight so the first batch's write-promotion + // pins release before the next allocation needs a slot. (This test was + // originally tuned to the deprecated max_write_batch_pages drain; the + // write budget expresses the same bound without the drain-to-zero.) + opts.max_inflight_write = 1; opts.auto_oom_retry_times = 0; auto build_entries = diff --git a/tests/io_qos.cpp b/tests/io_qos.cpp new file mode 100644 index 000000000..cf5b998ba --- /dev/null +++ b/tests/io_qos.cpp @@ -0,0 +1,405 @@ +/** + * IO QoS (docs/design/io_qos.md) — M1 in-flight page-IO budget tests. + * + * These tests run with deliberately tiny caps so the blocking paths are hot, + * then assert the accounting invariants: budgets drain to zero at quiesce, + * high-watermarks respect the caps (except the documented oversized-request + * admission), and disabled budgets stay untouched. + */ +#include +#include + +#include "async_io_manager.h" +#include "common.h" +#include "test_utils.h" + +using test_util::MapVerifier; + +namespace +{ +eloqstore::IoQosStats ShardStats(const eloqstore::EloqStore *store) +{ + return store->GetIoQosStats(0); +} +} // namespace + +TEST_CASE("io budgets: accounting invariants under tiny caps", "[io_qos]") +{ + eloqstore::KvOptions opts = default_opts; + opts.max_inflight_read = 4; + opts.max_inflight_write = 8; + eloqstore::EloqStore *store = InitStore(opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(200); + for (int round = 0; round < 4; round++) + { + verify.WriteRnd(0, 2000, 0, 25); + for (int i = 0; i < 100; i++) + { + verify.Read(std::rand() % 2000); + } + verify.Scan(0, 300); + } + + eloqstore::IoQosStats stats = ShardStats(store); + // Budgets drain to zero once all requests have completed. + REQUIRE(stats.read_.inflight_ == 0); + REQUIRE(stats.write_.inflight_ == 0); + // All page IO in this mode has cost 1, so watermarks are hard-capped. + REQUIRE(stats.read_.high_watermark_ >= 1); + REQUIRE(stats.read_.high_watermark_ <= 4); + REQUIRE(stats.write_.high_watermark_ >= 1); + REQUIRE(stats.write_.high_watermark_ <= 8); +} + +TEST_CASE("io budgets: overflow read batch larger than the cap", "[io_qos]") +{ + // 600KB values span ~150 overflow pages; with overflow_pointers = 128, + // GetOverflowValue issues 128-page ReadPages batches — far above the + // 4-page read cap. Per-page acquisition must make progress regardless. + eloqstore::KvOptions opts = default_opts; + opts.max_inflight_read = 4; + opts.max_inflight_write = 8; + opts.overflow_pointers = 128; + eloqstore::EloqStore *store = InitStore(opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(600 * 1024); + verify.Upsert(1); + verify.Upsert(2); + verify.Read(1); + verify.Read(2); + + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.read_.inflight_ == 0); + REQUIRE(stats.write_.inflight_ == 0); + // Cost-1 reads: the cap is strict even for oversized batches. + REQUIRE(stats.read_.high_watermark_ <= 4); + // A 128-page batch through a 4-page budget must have waited. + REQUIRE(stats.read_.blocked_count_ > 0); +} + +TEST_CASE("io budgets: merged append writes and oversized admission", + "[io_qos]") +{ + // Append mode aggregates page writes into ~1MB merged writes + // (cost = 256 pages at 4KB). With a 64-page write cap, each merged + // write exceeds the cap and is admitted alone once the budget drains: + // in-flight is bounded by the single-request cost, not the cap. + eloqstore::KvOptions opts = append_opts; + opts.max_inflight_read = 4; + opts.max_inflight_write = 64; + eloqstore::EloqStore *store = InitStore(opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(4000); + verify.WriteRnd(0, 3000, 0, 50); + for (int i = 0; i < 50; i++) + { + verify.Read(std::rand() % 3000); + } + + const uint32_t merged_cost_bound = + opts.write_buffer_size / opts.data_page_size; + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.read_.inflight_ == 0); + REQUIRE(stats.write_.inflight_ == 0); + REQUIRE(stats.write_.high_watermark_ >= 1); + REQUIRE(stats.write_.high_watermark_ <= merged_cost_bound); +} + +TEST_CASE("io budgets: cap 512 admits two concurrent merged writes", + "[io_qos]") +{ + // Counterpart of the oversized-admission test: with the cap at twice + // the merged-write cost (512 vs 256 pages), merged writes from + // concurrent write tasks may overlap in flight — the watermark must + // exceed one merged write's cost — while never exceeding the cap. + // (A single task's flushes do not reliably overlap: it yields per page + // while building the next buffer, so concurrency comes from multiple + // partitions' write tasks on one shard.) + eloqstore::KvOptions opts = append_opts; + opts.num_threads = 1; + opts.max_inflight_write = 512; + eloqstore::EloqStore *store = InitStore(opts); + + constexpr uint32_t num_parts = 8; + constexpr uint32_t keys_per_part = 1200; // ~1200 pages = ~5 flushes + const uint64_t ts = utils::UnixTs(); + std::array reqs; + std::atomic done{0}; + for (uint32_t p = 0; p < num_parts; p++) + { + std::vector entries; + entries.reserve(keys_per_part); + for (uint32_t i = 0; i < keys_per_part; i++) + { + entries.emplace_back(test_util::Key(i, 7), + std::string(3000, 'w'), + ts, + eloqstore::WriteOp::Upsert); + } + reqs[p].SetArgs(eloqstore::TableIdent("qos-dual", p), + std::move(entries)); + store->ExecAsyn(&reqs[p], + 0, + [&done](eloqstore::KvRequest *) + { done.fetch_add(1, std::memory_order_relaxed); }); + } + while (done.load(std::memory_order_relaxed) < int(num_parts)) + { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + for (auto &req : reqs) + { + REQUIRE(req.Error() == eloqstore::KvError::NoError); + } + + const uint32_t merged_cost = + opts.write_buffer_size / opts.data_page_size; // 256 + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.write_.inflight_ == 0); + REQUIRE(stats.write_.high_watermark_ > merged_cost); + REQUIRE(stats.write_.high_watermark_ <= 512); +} + +TEST_CASE("io budgets: failed write drains the budget", "[io_qos]") +{ + // Mid-batch write failure: make the partition directory read-only after + // the first data file exists, then write a batch large enough to need a + // second file. Creating that file fails (EACCES) with merged writes to + // the first file still in flight; the task aborts, and AbortWrite's + // WaitIo must drain every in-flight page so the budgets return to zero + // and the store stays usable. + namespace fs = std::filesystem; + eloqstore::KvOptions opts = append_opts; // 1MB files (2^8 pages) + eloqstore::EloqStore *store = InitStore(opts); + + eloqstore::TableIdent tbl_id{"qos-fail", 0}; + MapVerifier verify(tbl_id, store, false); + verify.SetValueSize(3000); // one KV per page + verify.Upsert(0, 10); // creates the partition dir + file 0 + + const fs::path part_dir = fs::path(test_path) / tbl_id.ToString(); + REQUIRE(fs::exists(part_dir)); + fs::permissions(part_dir, + fs::perms::owner_read | fs::perms::owner_exec, + fs::perm_options::replace); + + // ~600 pages: fills file 0 (256 pages) and needs file 1 -> EACCES. + std::vector entries; + const uint64_t ts = utils::UnixTs(); + for (uint32_t i = 100; i < 700; i++) + { + entries.emplace_back(std::to_string(1000000 + i), + std::string(3000, 'x'), + ts, + eloqstore::WriteOp::Upsert); + } + eloqstore::BatchWriteRequest fail_req; + fail_req.SetArgs(tbl_id, std::move(entries)); + store->ExecSync(&fail_req); + fs::permissions(part_dir, + fs::perms::owner_all, + fs::perm_options::replace); + REQUIRE(fail_req.Error() != eloqstore::KvError::NoError); + + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.write_.inflight_ == 0); + REQUIRE(stats.read_.inflight_ == 0); + REQUIRE(stats.bg_read_.inflight_ == 0); + + // The store must remain usable after the abort. + eloqstore::TableIdent recover_tbl{"qos-fail", 1}; + MapVerifier recover(recover_tbl, store, false); + recover.SetValueSize(200); + recover.Upsert(0, 50); + recover.Read(7); +} + +TEST_CASE("io budgets: shutdown while tasks queue behind the budget", + "[io_qos]") +{ + // Several overflow reads (128-page batches) contend for a 1-page read + // budget, then the store is stopped while they are still queued. Stop + // must drain cleanly (no hang, no crash) and every request must + // complete. + eloqstore::KvOptions opts = default_opts; + opts.max_inflight_read = 1; // bg_cap clamps to 1 as well + opts.overflow_pointers = 128; + eloqstore::EloqStore *store = InitStore(opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(600 * 1024); + verify.Upsert(0, 4); + + std::array reqs; + std::array keys; + std::atomic done{0}; + for (uint32_t i = 0; i < reqs.size(); i++) + { + keys[i] = test_util::Key(i, 7); // MapVerifier's key format + reqs[i].SetArgs(test_tbl_id, keys[i]); + store->ExecAsyn(&reqs[i], + 0, + [&done](eloqstore::KvRequest *) + { done.fetch_add(1, std::memory_order_relaxed); }); + } + store->Stop(); // blocks until the shard drains + + REQUIRE(done.load() == 4); + for (auto &req : reqs) + { + REQUIRE(req.Error() == eloqstore::KvError::NoError); + } + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.read_.inflight_ == 0); + REQUIRE(stats.write_.inflight_ == 0); +} + +TEST_CASE("io budgets: disabled read budget stays untouched", "[io_qos]") +{ + eloqstore::KvOptions opts = default_opts; + opts.max_inflight_read = 0; // disabled + eloqstore::EloqStore *store = InitStore(opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(200); + verify.WriteRnd(0, 1000, 0, 25); + for (int i = 0; i < 50; i++) + { + verify.Read(std::rand() % 1000); + } + + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.read_.inflight_ == 0); + REQUIRE(stats.read_.high_watermark_ == 0); + REQUIRE(stats.read_.blocked_count_ == 0); + // The write budget (default 32768) still counts, it just never blocks. + REQUIRE(stats.write_.inflight_ == 0); + REQUIRE(stats.write_.blocked_count_ == 0); +} + +TEST_CASE("bg sub-budget: compaction batch reads are bounded", "[io_qos]") +{ + // Append mode with full-overwrite rounds drives space amplification past + // file_amplify_factor, so the shard schedules compaction between batch + // writes (per-table writes serialize behind the internal compact + // request, so by the time the last sync write returns, earlier + // compactions have completed). Compaction move batches issue up to + // 128-page ReadPages bursts from a BackgroundWrite task — the + // BaseReqPageRead BG path — which must stay within the BG sub-budget + // (25% of 8 = 2 pages) while foreground keeps the full budget. + eloqstore::KvOptions opts = append_opts; + opts.file_amplify_factor = 2; + opts.max_inflight_read = 8; + opts.bg_read_ratio = 25; // bg_cap = 2 + eloqstore::EloqStore *store = InitStore(opts); + + MapVerifier verify(test_tbl_id, store, false); + // ~3000B values → one KV per 4KB data page, so key granularity equals + // page granularity and the overwrite pattern below controls per-file + // liveness exactly. (A fully-overwritten file is simply dropped by + // compaction with no page moves — the strided pattern keeps every file + // 40% live, i.e. SAF 2.5 > file_amplify_factor, forcing real moves.) + verify.SetValueSize(3000); + constexpr uint64_t num_keys = 1000; + verify.Upsert(0, num_keys); + for (int round = 0; round < 2; round++) + { + // Overwrite 3 of every 5 pages, uniformly across all files. + for (uint64_t base = 0; base < num_keys; base += 5) + { + verify.Upsert(base, base + 3); + } + } + for (int i = 0; i < 50; i++) + { + verify.Read(std::rand() % num_keys); + } + + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.read_.inflight_ == 0); + REQUIRE(stats.bg_read_.inflight_ == 0); + REQUIRE(stats.write_.inflight_ == 0); + // Compaction ran and its reads were charged to the BG class... + REQUIRE(stats.bg_read_.high_watermark_ >= 1); + // ...and never exceeded the sub-budget. + REQUIRE(stats.bg_read_.high_watermark_ <= 2); + // A 128-page move batch through a 2-page sub-budget must have waited. + REQUIRE(stats.bg_read_.blocked_count_ > 0); + // Total budget still respected. + REQUIRE(stats.read_.high_watermark_ <= 8); +} + +TEST_CASE("bg sub-budget: foreground reads use the full budget", "[io_qos]") +{ + // Foreground overflow reads (128-page batches) may exceed the BG cap and + // climb to the full read budget; only background is confined. + eloqstore::KvOptions opts = default_opts; + opts.max_inflight_read = 8; + opts.bg_read_ratio = 25; // bg_cap = 2 + opts.overflow_pointers = 128; + eloqstore::EloqStore *store = InitStore(opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(600 * 1024); + verify.Upsert(1); // fresh key: no tree reads, so no BG read traffic + verify.Read(1); // FG: 128-page overflow batches through cap 8 + + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.read_.inflight_ == 0); + // FG climbed past the BG cap — proof the sub-budget does not bind FG. + REQUIRE(stats.read_.high_watermark_ > 2); + REQUIRE(stats.read_.high_watermark_ <= 8); + // Nothing was charged to the BG class. + REQUIRE(stats.bg_read_.high_watermark_ == 0); + REQUIRE(stats.bg_read_.blocked_count_ == 0); +} + +TEST_CASE("bg sub-budget: batch-write leaf loads are background", + "[io_qos]") +{ + // Overwriting existing keys forces the BatchWrite task to load leaf data + // pages from disk (single-page KvTaskPageRead path). BatchWrite is + // classified background, so those loads are charged to the sub-budget. + eloqstore::KvOptions opts = default_opts; + opts.max_inflight_read = 8; + opts.bg_read_ratio = 25; // bg_cap = 2 + eloqstore::EloqStore *store = InitStore(opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(200); + verify.WriteRnd(0, 2000, 0, 100); // initial load + verify.WriteRnd(0, 2000, 0, 100); // overwrite: leaf loads from disk + + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.read_.inflight_ == 0); + REQUIRE(stats.bg_read_.inflight_ == 0); + REQUIRE(stats.bg_read_.high_watermark_ >= 1); + REQUIRE(stats.bg_read_.high_watermark_ <= 2); +} + +TEST_CASE("io budgets: defaults are behavior-neutral", "[io_qos]") +{ + // At default caps (read 32, write 32768) a single-threaded unit + // workload of small values must never block on a budget: foreground + // point reads are sequential (in-flight 1) and batch-write leaf loads + // are sequential background singles, well under bg_cap = 8. + eloqstore::EloqStore *store = InitStore(default_opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(200); + verify.WriteRnd(0, 2000, 0, 25); + for (int i = 0; i < 100; i++) + { + verify.Read(std::rand() % 2000); + } + + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.read_.inflight_ == 0); + REQUIRE(stats.write_.inflight_ == 0); + REQUIRE(stats.read_.blocked_count_ == 0); + REQUIRE(stats.write_.blocked_count_ == 0); +} From 052034c03a4846d8625b06572b9b23509415d9e5 Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Fri, 3 Jul 2026 20:16:09 -0700 Subject: [PATCH 03/30] Fix format of testing code. --- benchmark/interference_bench.cpp | 28 +++++++++++++--------------- tests/io_qos.cpp | 10 +++------- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/benchmark/interference_bench.cpp b/benchmark/interference_bench.cpp index 3916b29c1..e0e583f31 100644 --- a/benchmark/interference_bench.cpp +++ b/benchmark/interference_bench.cpp @@ -419,26 +419,24 @@ void ReportQosDelta(const char *name, : 0.0; }; LOG(INFO) << "RESULT qos phase=" << name << " shard=" << shard - << " read_hwm=" << end.read_.high_watermark_ - << " read_blocked=" << d(begin.read_.blocked_count_, - end.read_.blocked_count_) - << " read_blocked_us=" << d(begin.read_.blocked_us_, - end.read_.blocked_us_) + << " read_hwm=" << end.read_.high_watermark_ << " read_blocked=" + << d(begin.read_.blocked_count_, end.read_.blocked_count_) + << " read_blocked_us=" + << d(begin.read_.blocked_us_, end.read_.blocked_us_) << " read_page_mbps=" << mbps(begin.read_, end.read_) << " bg_read_hwm=" << end.bg_read_.high_watermark_ - << " bg_read_blocked=" << d(begin.bg_read_.blocked_count_, - end.bg_read_.blocked_count_) - << " bg_read_blocked_us=" << d(begin.bg_read_.blocked_us_, - end.bg_read_.blocked_us_) + << " bg_read_blocked=" + << d(begin.bg_read_.blocked_count_, end.bg_read_.blocked_count_) + << " bg_read_blocked_us=" + << d(begin.bg_read_.blocked_us_, end.bg_read_.blocked_us_) << " bg_read_page_mbps=" << mbps(begin.bg_read_, end.bg_read_) << " write_hwm=" << end.write_.high_watermark_ - << " write_blocked=" << d(begin.write_.blocked_count_, - end.write_.blocked_count_) + << " write_blocked=" + << d(begin.write_.blocked_count_, end.write_.blocked_count_) << " write_page_mbps=" << mbps(begin.write_, end.write_) - << " fdatasync=" << d(begin.fdatasync_count_, - end.fdatasync_count_) - << " fdatasync_us=" << d(begin.fdatasync_us_, - end.fdatasync_us_); + << " fdatasync=" + << d(begin.fdatasync_count_, end.fdatasync_count_) + << " fdatasync_us=" << d(begin.fdatasync_us_, end.fdatasync_us_); } } // namespace diff --git a/tests/io_qos.cpp b/tests/io_qos.cpp index cf5b998ba..9c025c85c 100644 --- a/tests/io_qos.cpp +++ b/tests/io_qos.cpp @@ -109,8 +109,7 @@ TEST_CASE("io budgets: merged append writes and oversized admission", REQUIRE(stats.write_.high_watermark_ <= merged_cost_bound); } -TEST_CASE("io budgets: cap 512 admits two concurrent merged writes", - "[io_qos]") +TEST_CASE("io budgets: cap 512 admits two concurrent merged writes", "[io_qos]") { // Counterpart of the oversized-admission test: with the cap at twice // the merged-write cost (512 vs 256 pages), merged writes from @@ -200,9 +199,7 @@ TEST_CASE("io budgets: failed write drains the budget", "[io_qos]") eloqstore::BatchWriteRequest fail_req; fail_req.SetArgs(tbl_id, std::move(entries)); store->ExecSync(&fail_req); - fs::permissions(part_dir, - fs::perms::owner_all, - fs::perm_options::replace); + fs::permissions(part_dir, fs::perms::owner_all, fs::perm_options::replace); REQUIRE(fail_req.Error() != eloqstore::KvError::NoError); eloqstore::IoQosStats stats = ShardStats(store); @@ -358,8 +355,7 @@ TEST_CASE("bg sub-budget: foreground reads use the full budget", "[io_qos]") REQUIRE(stats.bg_read_.blocked_count_ == 0); } -TEST_CASE("bg sub-budget: batch-write leaf loads are background", - "[io_qos]") +TEST_CASE("bg sub-budget: batch-write leaf loads are background", "[io_qos]") { // Overwriting existing keys forces the BatchWrite task to load leaf data // pages from disk (single-page KvTaskPageRead path). BatchWrite is From 950a4f4a28d4678dc6746324e3ce2c6aab873b46 Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Fri, 3 Jul 2026 21:38:52 -0700 Subject: [PATCH 04/30] Benchmark-tagged Catch2 cases are now hidden ([.]) so CI's ctest run skips them; select the [benchmark] tag to run them explicitly. --- tests/large_value_benchmark.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/large_value_benchmark.cpp b/tests/large_value_benchmark.cpp index 8823c0915..df7a7547a 100644 --- a/tests/large_value_benchmark.cpp +++ b/tests/large_value_benchmark.cpp @@ -1,7 +1,9 @@ // tests/large_value_benchmark.cpp // // Informational benchmarks for the zero-copy large-value read/write paths. -// Excluded from the normal (non-benchmark) test run. +// Excluded from the normal test run via the Catch2 hidden tag "[.]": not +// discovered by ctest/CI and skipped on a bare binary run; select the tag +// explicitly to execute. // // Run with: // sudo prlimit --memlock=unlimited ./large_value_benchmark -t '[benchmark]' @@ -477,7 +479,7 @@ constexpr ConcurrencyConfig kConcurrencyConfigs[] = { // inference server with multiple in-flight requests) would see. // ============================================================================ TEST_CASE("Benchmark: concurrent random reads (large-value vs overflow-page)", - "[benchmark][read-latency]") + "[.][benchmark][read-latency]") { constexpr size_t kTargetDataset = 256ULL * 1024 * 1024; constexpr int kMinKeys = 256; @@ -661,7 +663,7 @@ TEST_CASE("Benchmark: concurrent random reads (large-value vs overflow-page)", // std::string path writes overflow pages via regular io_uring writes. // ============================================================================ TEST_CASE("Benchmark: batch-write throughput IoStringBuffer vs std::string", - "[benchmark][write-throughput]") + "[.][benchmark][write-throughput]") { BenchHarness h; eloqstore::KvOptions opts = MakeBenchOpts(h); @@ -834,7 +836,7 @@ TEST_CASE("Benchmark: batch-write throughput IoStringBuffer vs std::string", // Expected: smaller yield_every → more frequent yields → lower p99 overhead. // ============================================================================ TEST_CASE("Benchmark: foreground read p99 during segment compaction", - "[benchmark][compact-overhead]") + "[.][benchmark][compact-overhead]") { const uint32_t yield_values[] = {1, 8, 32}; @@ -1222,7 +1224,7 @@ std::vector TimePinnedReadsOverloadC(eloqstore::EloqStore *store, // throughput; the kernel just reads the same source range M times. // ============================================================================ TEST_CASE("Benchmark: pinned-mode batch-write throughput", - "[benchmark][write-throughput][pinned]") + "[.][benchmark][write-throughput][pinned]") { struct SizeCase { @@ -1429,7 +1431,7 @@ TEST_CASE("Benchmark: pinned-mode batch-write throughput", // scratch acquire/release. // ============================================================================ TEST_CASE("Benchmark: pinned-write tail-scratch overhead", - "[benchmark][write-latency][pinned][tail-scratch]") + "[.][benchmark][write-latency][pinned][tail-scratch]") { struct SizeCase { @@ -1545,7 +1547,7 @@ TEST_CASE("Benchmark: pinned-write tail-scratch overhead", // aggregate MB/s (total bytes / wall time). // ============================================================================ TEST_CASE("Benchmark: pinned-mode concurrent random reads", - "[benchmark][read-latency][pinned]") + "[.][benchmark][read-latency][pinned]") { constexpr size_t kTargetDataset = 256ULL * 1024 * 1024; constexpr int kMinKeys = 256; @@ -1677,7 +1679,7 @@ TEST_CASE("Benchmark: pinned-mode concurrent random reads", // window saves the index navigation on the (W-1)/W subsequent reads. // ============================================================================ TEST_CASE("Benchmark: pinned-mode concurrent windowed-locality reads", - "[benchmark][read-latency][pinned]") + "[.][benchmark][read-latency][pinned]") { constexpr size_t kTargetDataset = 256ULL * 1024 * 1024; constexpr int kMinKeys = 256; From faea3a46352834f758dd64e1edda38d5ab5ac22c Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Wed, 15 Jul 2026 21:44:47 -0700 Subject: [PATCH 05/30] fix(io): FG read-budget wake starvation, idle-CQE stalls; raise default read budget to 64; add IO stage-timing instrumentation and GET2 bench MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two behavioral fixes found during the Azure NVMe calibration campaign (2026-07-11), one retuned default, and supporting instrumentation: - IoBudget::Release: always wake the foreground class, not only with leftover credits. Under a write storm, background forms a saturated treadmill whose wait-queue never empties, so every released credit re-donated to background; once the last foreground in-flight's credit landed in a background dip, the foreground class had no remaining wake source until background drained — observed as multi-second foreground gate stalls (p999 120–312 ms, gate waits up to 3.7 s). Over-waking is safe: woken tasks re-check the admission condition and re-wait. - IouringMgr::IsIdle: track in-flight SQEs (prepared minus reaped) and report non-idle while any remain. The base-class IsIdle let shards sleep up to the 100 ms request-wait timeout with CQEs pending, which stalls delivery under DEFER_TASKRUN when the pending IO is not owned by an active task (observed during prewarm). CloudStoreMgr::IsIdle now composes the base check. - Raise default max_inflight_read 32 -> 64 (header + INI fallback). Azure local-NVMe calibration put the knee at c ~= 5-7 x BDP: 64–128 indistinguishable, while 32 showed genuine foreground queueing under 128 concurrent readers — the budget must also cover peak per-shard foreground concurrency, not only the BDP. bg_read_ratio stays 25; docs/design/io_qos.md records the calibration and warns that ratio x cap combinations yielding bg_cap < ~8 throttle ingest itself (batch-write RMW page fetches ride the background class). - Opt-in stage-timing instrumentation, gated by ELOQ_IO_STATS=1 at runtime (off: one cached-bool branch per site): per-request enqueue/dequeue stamps, read-path stage breakdown (budget gate, SQE->CQE, CQE->resume, queue wait, start lag, index walk), per-loop phase timing with SLOWROUND reports for >1 ms rounds, and 5 s opstages/loopstats VLOG(1) summaries with CQE reap-batch histogram. - benchmark: new GET2 mode — dedicated client threads each holding --inflight_per_client async reads, optional --per_shard_cap to bound a stalled shard's blast radius; reports QPS and latency percentiles. - object_store: suppress libcurl's default form-urlencoded Content-Type on presigned-URL uploads; it is outside the signature (SignedHeaders=host) and strict validators (s3proxy) reject it. Also: Shard::ReadTimeMicroseconds is now static (callers need no instance). --- benchmark/eloq_store_bm.cc | 188 +++++++++++++++++++++++++++++++++++ benchmark/eloq_store_bm.h | 11 ++ benchmark/main.cpp | 8 ++ docs/design/io_qos.md | 38 +++++-- include/async_io_manager.h | 37 +++++++ include/eloq_store.h | 4 + include/kv_options.h | 2 +- include/storage/shard.h | 2 +- include/tasks/task.h | 8 ++ include/utils.h | 11 ++ src/async_io_manager.cpp | 135 ++++++++++++++++++++++++- src/eloq_store.cpp | 4 + src/kv_options.cpp | 2 +- src/storage/object_store.cpp | 4 + src/storage/shard.cpp | 68 ++++++++++--- src/tasks/read_task.cpp | 4 + 16 files changed, 500 insertions(+), 26 deletions(-) diff --git a/benchmark/eloq_store_bm.cc b/benchmark/eloq_store_bm.cc index 2c066dc06..2923ffb72 100644 --- a/benchmark/eloq_store_bm.cc +++ b/benchmark/eloq_store_bm.cc @@ -1,11 +1,24 @@ #include "eloq_store_bm.h" +#include + +#include +#include #include +#include #include #include +#include +// https://github.com/cameron314/concurrentqueue/issues/280 +#undef BLOCK_SIZE +#include "../external/concurrentqueue/blockingconcurrentqueue.h" #include "kv_options.h" +DECLARE_uint32(client_threads); +DECLARE_uint32(inflight_per_client); +DECLARE_uint32(per_shard_cap); + namespace EloqStoreBM { static const std::string table_name_str = "eloq_store_bm"; @@ -530,6 +543,172 @@ void Benchmark::GenBatchRecord(const Benchmark &bm, #endif } +namespace +{ +struct Get2Client +{ + moodycamel::BlockingConcurrentQueue done_; + std::vector lat_us_; + uint64_t completed_{0}; + uint64_t issue_failed_{0}; +}; + +uint64_t Get2NowUs() +{ + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} +} // namespace + +void Benchmark::OnReadV2(::eloqstore::KvRequest *req) +{ + auto *op = reinterpret_cast(req->UserData()); + static_cast(op->client_)->done_.enqueue(op); +} + +void Benchmark::RunGet2(uint32_t client_threads, + uint32_t inflight, + uint32_t per_shard_cap) +{ + const uint32_t nshards = worker_cnt_ > 0 ? worker_cnt_ : 1; + std::atomic stop{false}; + std::vector clients(client_threads); + std::vector thds; + const uint64_t bench_start = Get2NowUs(); + + for (uint32_t c = 0; c < client_threads; ++c) + { + thds.emplace_back( + [this, c, inflight, per_shard_cap, nshards, &clients, &stop]() + { + Get2Client &me = clients[c]; + object_generator gen; + gen.set_random_data(true); + gen.set_random_seed(20260711 + c * 7919); + gen.set_key_size(key_byte_size_); + gen.set_data_size_fixed(value_byte_size_); + gen.set_key_prefix(key_prefix_.data()); + gen.set_key_range(key_minimum_, key_maximum_); + + std::vector ops; + ops.reserve(inflight); + for (uint32_t i = 0; i < inflight; ++i) + { + ops.emplace_back(this); + ops.back().client_ = &me; + } + std::vector shard_out(nshards, 0); + uint64_t issued = 0; + + auto issue = [&](ReadOperation *op) -> bool + { + uint64_t key_index = + gen.get_key_index(OBJECT_GENERATOR_KEY_RANDOM); + uint32_t part = key_index % partition_count_; + if (per_shard_cap > 0) + { + for (int tries = 0; + shard_out[part % nshards] >= per_shard_cap && + tries < 8; + ++tries) + { + key_index = + gen.get_key_index(OBJECT_GENERATOR_KEY_RANDOM); + part = key_index % partition_count_; + } + } + op->shard_ = part % nshards; + op->key_.clear(); + gen.generate_key(key_index, op->key_); + op->req_->SetArgs( + ::eloqstore::TableIdent(table_name_str, part), + op->key_); + op->start_ts_ = Get2NowUs(); + if (!eloq_store_->ExecAsyn(op->req_.get(), + reinterpret_cast(op), + OnReadV2)) + { + ++me.issue_failed_; + return false; + } + ++shard_out[op->shard_]; + ++issued; + return true; + }; + + for (auto &op : ops) + { + issue(&op); + } + ReadOperation *done_op = nullptr; + while (!stop.load(std::memory_order_relaxed)) + { + if (!me.done_.wait_dequeue_timed(done_op, 10000)) + { + continue; + } + const uint64_t now = Get2NowUs(); + me.lat_us_.push_back(now - done_op->start_ts_); + ++me.completed_; + --shard_out[done_op->shard_]; + issue(done_op); + } + // Drain remaining in-flight before exiting. + uint64_t drained = me.completed_; + const uint64_t deadline = Get2NowUs() + 3000000; + while (drained + me.issue_failed_ < issued && + Get2NowUs() < deadline) + { + if (me.done_.wait_dequeue_timed(done_op, 10000)) + { + ++drained; + } + } + }); + } + + std::this_thread::sleep_for(std::chrono::seconds(total_test_time_sec_)); + stop.store(true, std::memory_order_release); + for (auto &t : thds) + { + t.join(); + } + const double dur_sec = (Get2NowUs() - bench_start) / 1e6; + + std::vector all; + uint64_t total = 0; + for (auto &cl : clients) + { + total += cl.completed_; + all.insert(all.end(), cl.lat_us_.begin(), cl.lat_us_.end()); + } + std::sort(all.begin(), all.end()); + auto pct = [&](double p) -> uint64_t + { + if (all.empty()) + { + return 0; + } + size_t idx = + std::min(all.size() - 1, static_cast(p * all.size())); + return all[idx]; + }; + LOG(INFO) << "GET2 finished: clients=" << client_threads + << " inflight=" << inflight << " per_shard_cap=" << per_shard_cap + << " completed=" << total << " duration=" << dur_sec + << "s QPS:" << std::fixed << std::setprecision(2) + << total / dur_sec; + LOG(INFO) << "Latency: Min->" << (all.empty() ? 0 : all.front()) + << ", Max->" << (all.empty() ? 0 : all.back()) << ", Mean->" + << (all.empty() ? 0 + : std::accumulate(all.begin(), all.end(), 0ULL) / + all.size()) + << ", p50->" << pct(0.50) << ", p90->" << pct(0.90) << ", p95->" + << pct(0.95) << ", p99->" << pct(0.99) << ", p99.9->" + << pct(0.999) << ", p99.99->" << pct(0.9999); +} + void Benchmark::OnRead(::eloqstore::KvRequest *req) { ::eloqstore::ReadRequest *read_req = @@ -611,6 +790,7 @@ void Benchmark::OnRead(::eloqstore::KvRequest *req) // get next key randomly. int8_t iter = obj_iter_type(read_op->bm_->key_pattern_, GET_CMD_IDX); uint64_t key_index = read_obj_gen.get_key_index(iter); + read_op->key_.clear(); read_obj_gen.generate_key(key_index, read_op->key_); @@ -725,6 +905,7 @@ Benchmark::Benchmark(std::string &command, key_pattern_(key_pattern), result_(worker_cnt, this) { + worker_cnt_ = worker_cnt; } bool Benchmark::OpenEloqStore(const eloqstore::KvOptions &kv_options) @@ -848,6 +1029,13 @@ void Benchmark::RunBenchmark() } } } + else if (command_ == "GET2") + { + RunGet2(FLAGS_client_threads, + FLAGS_inflight_per_client, + FLAGS_per_shard_cap); + return; + } else { LOG(ERROR) << "Unsupport command: " << command_; diff --git a/benchmark/eloq_store_bm.h b/benchmark/eloq_store_bm.h index f1fb8ed8a..9f35273d4 100644 --- a/benchmark/eloq_store_bm.h +++ b/benchmark/eloq_store_bm.h @@ -144,6 +144,9 @@ struct ReadOperation std::string key_; uint64_t start_ts_{0}; const Benchmark *bm_{nullptr}; + // GET2 mode: owning client and target shard of the in-flight request. + void *client_{nullptr}; + uint32_t shard_{0}; }; class BMResult @@ -220,6 +223,13 @@ class Benchmark void CloseEloqStore(); void RunBenchmark(); + // GET2: dedicated client threads, each keeping `inflight` async reads + // outstanding; optional per-shard outstanding cap bounds the blast + // radius of a stalled shard. + void RunGet2(uint32_t client_threads, + uint32_t inflight, + uint32_t per_shard_cap); + static void OnReadV2(::eloqstore::KvRequest *req); private: static void OnBatchWrite(::eloqstore::KvRequest *req); @@ -239,6 +249,7 @@ class Benchmark std::string command_; size_t total_data_size_{0}; const uint32_t partition_count_{0}; + uint32_t worker_cnt_{0}; // num shard threads (for same-shard mode) uint32_t key_byte_size_{0}; uint32_t value_byte_size_{0}; std::string key_prefix_; diff --git a/benchmark/main.cpp b/benchmark/main.cpp index a75ed366d..988e3a40b 100644 --- a/benchmark/main.cpp +++ b/benchmark/main.cpp @@ -36,6 +36,14 @@ DEFINE_uint32(batch_size, 64, "The batch size of one write request (MB) (default: 64)."); DEFINE_string(storage, "eloqstore", "The storage used to store the data."); +DEFINE_uint32(client_threads, 4, "GET2: number of client threads."); +DEFINE_uint32(inflight_per_client, + 125, + "GET2: async requests each client keeps in flight."); +DEFINE_uint32(per_shard_cap, + 0, + "GET2: max outstanding per shard per client (0 = unlimited); " + "bounds the blast radius of a stalled shard."); DEFINE_uint32(request_cnt, 32, "The number of concurrent read requests (default: 32)"); diff --git a/docs/design/io_qos.md b/docs/design/io_qos.md index b84346fd9..9b2a39c3b 100644 --- a/docs/design/io_qos.md +++ b/docs/design/io_qos.md @@ -196,8 +196,17 @@ foreground reclaims the whole budget the moment background demand drains. Two read `WaitingZone`s (one per class, FIFO within class). On release, background waiters are woken first while the sub-budget has room (those -units are reserved for them anyway); unused wake credits are forwarded to -foreground (`WaitingZone::WakeN` returns the number actually woken). +units are reserved for them anyway), and foreground is **always** woken as +well — not merely handed leftover credits. The leftover-only scheme shipped +first and starved the foreground class under write storms (found 2026-07-11 +on Azure NVMe): background forms a saturated treadmill whose queue never +empties, so every release re-donates its credit to background; once any +foreground read queued (admission is FIFO-behind-waiters) and the last +foreground in-flight's credit landed in a background dip, the foreground +class had **no remaining wake source** until background's queue drained — +observed as multi-second foreground gate stalls (p999 120–312 ms, gate +waits up to 3.7 s). Over-waking is safe by construction: woken tasks +re-check the admission condition and re-wait. The ratio starts **static**. An adaptive policy (shrink `bg_read_limit` toward a floor when foreground waiters exist, grow toward the full budget @@ -221,18 +230,17 @@ compaction, since interference tracks bytes/sec rather than queue depth. ### New options (per shard) ``` -uint32_t max_inflight_read = 32; // pages; 0 = disabled +uint32_t max_inflight_read = 64; // pages; 0 = disabled uint32_t bg_read_ratio = 25; // percent of max_inflight_read uint32_t max_inflight_write = 512; // pages; REDEFINED existing option // (was 32768, pool sizing only) uint64_t bg_write_rate_limit = 0; // bytes/sec; 0 = disabled (M3) ``` -max_inflight_read = 32 and bg_read_ratio = 25 were set from the WSL -interference sweeps (2026-07-03): total caps 32–256 were indistinguishable -at fixed bg_cap, 32 matched that box's BDP and gave the campaign's best -p50/p999, and bg_cap 8 (25% of 32) sat exactly at the measured background -demand line — tail protection at zero write-throughput cost (the +bg_read_ratio = 25 was set from the WSL interference sweeps (2026-07-03): +total caps 32–256 were indistinguishable at fixed bg_cap, 32 matched that +box's BDP and gave the campaign's best p50/p999, and bg_cap 8 (25% of 32) +sat exactly at the measured background demand line — tail protection at zero write-throughput cost (the write-throughput knee appeared only at bg_cap 4). Real-device calibration (the QD sweep below) should re-derive max_inflight_read per device; the ratio is policy and should transfer. max_inflight_write's real default @@ -244,7 +252,12 @@ The two read-side options deliberately live at different levels: - **`max_inflight_read` is device calibration.** Size it from the device's bandwidth-delay product: `c × max_random_read_IOPS × t_read(unloaded) / - num_threads`, with c ≈ 2–4 for die-imbalance and burst headroom. Below + num_threads`, with c ≈ 2–4 for die-imbalance and burst headroom (Azure + local-NVMe calibration 2026-07-11 measured the knee at c ≈ 5–7: budgets + 64–128 for 16 shards on 8 × ~250K-IOPS devices at ~150 µs loaded latency, + indistinguishable within cloud-environment noise; 32 showed genuine + foreground queueing under 128 concurrent readers — the budget must also + cover peak per-shard foreground concurrency, not only the BDP). Below the BDP the cap costs read throughput; far above it the cap stops representing the device queue and the ratio contract below degrades. Validation signal: foreground `read_blocked` should stay ≈ 0 under @@ -265,6 +278,13 @@ The two read-side options deliberately live at different levels: remedy is raising `max_inflight_read` (the device has headroom the calibration missed), not the ratio — raising the ratio spends the tail contract. +- **The background slice needs an absolute floor.** Batch writes issue + their own read-modify-write page fetches under the background class, so + a tiny `ratio × cap` throttles ingest itself, not just compaction: + measured at bg_cap = 3 (ratio 10 of cap 32), write throughput halved and + the paced foreground collapsed with it. Until the engine enforces + `bg_cap = max(floor, ratio × cap)` (floor ≈ 8 on tested devices), do not + configure combinations that yield bg_cap below ~8. ## Interaction with Existing Knobs diff --git a/include/async_io_manager.h b/include/async_io_manager.h index 6f16819a7..8acfa6b3c 100644 --- a/include/async_io_manager.h +++ b/include/async_io_manager.h @@ -657,6 +657,16 @@ class IouringMgr : public AsyncIoManager ~IouringMgr() override; KvError Init(Shard *shard) override; void Submit() override; + // A shard must not idle-block while this ring has prepared or + // submitted-but-unreaped IO: CQE delivery under DEFER_TASKRUN + // requires this thread to keep entering the kernel. The base class + // returns true unconditionally, which let shards sleep up to the + // 100ms request-wait timeout with CQEs pending (observed during + // prewarm, whose IO is not owned by an active task). + bool IsIdle() override + { + return inflight_ios_ == 0; + } void PollComplete() override; char *AcquireWriteBuffer(uint16_t &buf_index) override; void ReleaseWriteBuffer(char *ptr, uint16_t buf_index) override; @@ -1179,6 +1189,33 @@ class IouringMgr : public AsyncIoManager // there's nothing to do. static constexpr uint32_t kForceSubmitEveryNoOps = 10; uint32_t consecutive_skipped_submits_{0}; + // SQEs handed out minus CQEs reaped (single shard thread). + uint32_t inflight_ios_{0}; + + // Loop-behavior stats, maintained by the owning shard thread and + // flushed as one VLOG(1) line every 5s: iteration rate, CQE rate, + // and the reap-batch-size histogram (how many CQEs each non-empty + // PollComplete found). Used to observe completion-batching regimes. + uint64_t loop_now_us_{0}; // stamped by Submit() each loop iteration + uint64_t stats_next_flush_us_{0}; + uint64_t stats_iters_{0}; + uint64_t stats_polls_nonzero_{0}; + uint64_t stats_cqes_{0}; + uint64_t stats_batch_hist_[9]{}; // index 1..8 = batch size, 0 = 9+ + // Per-stage read-path timing (ELOQ_IO_STATS=1): [0]=budget-gate wait, + // [1]=SQE->CQE (pre-dispatch + device + reap), [2]=CQE->task resume, + // [3]=enqueue->dequeue (queue wait), [4]=task-start->gate + // (index/root walk), [5]=dequeue->task-start (start lag). + bool io_stats_enabled_{false}; + uint64_t stage_sum_us_[6]{}; + uint64_t stage_max_us_[6]{}; + // Loop round-length tracking (gap between successive PollComplete + // calls on this shard) while stats are enabled. + uint64_t round_prev_us_{0}; + uint64_t round_sum_us_{0}; + uint64_t round_max_us_{0}; + uint64_t round_cnt_{0}; + uint64_t stage_cnt_{0}; // Active branch for this shard. std::string active_branch_{MainBranchName}; diff --git a/include/eloq_store.h b/include/eloq_store.h index bed220d94..7ce0c34c6 100644 --- a/include/eloq_store.h +++ b/include/eloq_store.h @@ -142,6 +142,10 @@ class KvRequest void SetTableId(TableIdent tbl_id); const TableIdent &TableId() const; uint64_t UserData() const; + // Stage-timing instrumentation only (ELOQ_IO_STATS=1): microsecond + // timestamp when SendRequest enqueued this request to its shard. + uint64_t dbg_enqueue_us_{0}; + uint64_t dbg_dequeue_us_{0}; /** * @brief Test if this request is done. diff --git a/include/kv_options.h b/include/kv_options.h index 6fd293be6..e252bc6e9 100644 --- a/include/kv_options.h +++ b/include/kv_options.h @@ -98,7 +98,7 @@ struct KvOptions * `read_blocked` staying ~0 under representative load validates the * value; nonzero means undersized. */ - uint32_t max_inflight_read = 32; + uint32_t max_inflight_read = 64; /** * @brief Background share of max_inflight_read, in percent (clamped to * 1..100; docs/design/io_qos.md M2). Page reads issued by background diff --git a/include/storage/shard.h b/include/storage/shard.h index 521fb5a79..9fb9cf28a 100644 --- a/include/storage/shard.h +++ b/include/storage/shard.h @@ -82,7 +82,7 @@ class Shard // counter on aarch64). Public so shard-thread code outside Shard (e.g. // IoBudget blocked-time accounting) can time intervals without a // clock_gettime call. - uint64_t ReadTimeMicroseconds(); + static uint64_t ReadTimeMicroseconds(); uint64_t DurationMicroseconds(uint64_t start_us); std::atomic io_mgr_and_page_pool_inited_{false}; diff --git a/include/tasks/task.h b/include/tasks/task.h index 5f063ef79..dd19d8709 100644 --- a/include/tasks/task.h +++ b/include/tasks/task.h @@ -231,6 +231,14 @@ class KvTask int io_res_{0}; uint32_t io_flags_{0}; KvError result_err_{KvError::NoError}; + // Loop-time (us) when this task's latest page-read CQE was reaped; + // stage-timing instrumentation only (ELOQ_IO_STATS=1). + uint64_t op_cqe_us_{0}; + // Stage-timing (ELOQ_IO_STATS=1): when this task began executing its + // read; consumed (zeroed) by the first instrumented page read. + uint64_t op_start_us_{0}; + bool needs_auto_reopen_{false}; + bool needs_oom_retry_{false}; TaskStatus status_{TaskStatus::Idle}; KvRequest *req_{nullptr}; diff --git a/include/utils.h b/include/utils.h index 7a22a33e8..d4449992e 100644 --- a/include/utils.h +++ b/include/utils.h @@ -215,6 +215,17 @@ inline std::string BuildStorePathListWithWeights( return oss.str(); } +// Runtime gate for the IO stage-timing instrumentation (ELOQ_IO_STATS=1). +inline bool IoStatsEnabled() +{ + static const bool enabled = [] + { + const char *e = getenv("ELOQ_IO_STATS"); + return e != nullptr && e[0] == '1'; + }(); + return enabled; +} + inline std::vector ComputeStorePathLut( const std::vector &weights, size_t max_entries = kDefaultStorePathLutEntries) diff --git a/src/async_io_manager.cpp b/src/async_io_manager.cpp index 199da7c99..45c7957dd 100644 --- a/src/async_io_manager.cpp +++ b/src/async_io_manager.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -256,13 +257,24 @@ void IoBudget::Release(uint32_t cost, bool background) { woken = bg_waiting_.WakeN(cost); } - waiting_.WakeN(cost - woken); + // Always give foreground a wake as well, not only the leftover + // credits. When background is a saturated treadmill (its queue never + // empties, so every release re-donates to background), foreground + // waiters would otherwise have no wake source once the last + // foreground in-flight completes: the class deadlocks behind + // `!zone.Empty()` admission until background's queue happens to + // drain (observed as multi-hundred-ms foreground gate stalls under + // write storms). Over-waking is safe: woken tasks re-check the + // admission condition and re-wait. + waiting_.WakeN(cost > woken ? cost - woken : 1); } IouringMgr::IouringMgr(const KvOptions *opts, uint32_t fd_limit) : AsyncIoManager(opts), fd_limit_(fd_limit) { memset(&ring_, 0, sizeof(ring_)); + const char *iostats_env = getenv("ELOQ_IO_STATS"); + io_stats_enabled_ = iostats_env != nullptr && iostats_env[0] == '1'; lru_fd_head_.next_ = &lru_fd_tail_; lru_fd_tail_.prev_ = &lru_fd_head_; @@ -854,7 +866,11 @@ std::pair IouringMgr::ReadPage(const TableIdent &tbl_id, // before SQE prep; released per CQE in PollComplete, so each // retry iteration re-acquires. Background tasks are additionally // bounded by the BG sub-budget. + const uint64_t t_gate = + io_stats_enabled_ ? shard->ReadTimeMicroseconds() : 0; read_budget_.Acquire(1, ThdTask()->IsBackground()); + const uint64_t t_sqe = + io_stats_enabled_ ? shard->ReadTimeMicroseconds() : 0; io_uring_sqe *sqe = GetSQE(UserDataType::KvTaskPageRead, ThdTask()); if (fd.second) { @@ -878,6 +894,46 @@ std::pair IouringMgr::ReadPage(const TableIdent &tbl_id, sqe, fd.first, dst, options_->data_page_size, offset); } res = ThdTask()->WaitIoResult(); + if (io_stats_enabled_) + { + const uint64_t t_res = shard->ReadTimeMicroseconds(); + const uint64_t cqe = ThdTask()->op_cqe_us_; + const uint64_t t_task = ThdTask()->op_start_us_; + if (t_task != 0) + { + ThdTask()->op_start_us_ = 0; // first page read only + KvRequest *r = ThdTask()->req_; + if (r != nullptr && r->dbg_enqueue_us_ != 0 && + r->dbg_dequeue_us_ >= r->dbg_enqueue_us_ && + t_task >= r->dbg_dequeue_us_) + { + const uint64_t d3 = + r->dbg_dequeue_us_ - r->dbg_enqueue_us_; + const uint64_t d5 = t_task - r->dbg_dequeue_us_; + stage_sum_us_[3] += d3; + stage_max_us_[3] = std::max(stage_max_us_[3], d3); + stage_sum_us_[5] += d5; + stage_max_us_[5] = std::max(stage_max_us_[5], d5); + } + if (t_gate > t_task) + { + const uint64_t d4 = t_gate - t_task; + stage_sum_us_[4] += d4; + stage_max_us_[4] = std::max(stage_max_us_[4], d4); + } + } + const uint64_t d0 = t_sqe - t_gate; // gate wait + const uint64_t d1 = cqe > t_sqe ? cqe - t_sqe : 0; // sqe->cqe + const uint64_t d2 = + t_res > cqe ? t_res - cqe : 0; // cqe->resume + stage_sum_us_[0] += d0; + stage_sum_us_[1] += d1; + stage_sum_us_[2] += d2; + stage_max_us_[0] = std::max(stage_max_us_[0], d0); + stage_max_us_[1] = std::max(stage_max_us_[1], d1); + stage_max_us_[2] = std::max(stage_max_us_[2], d2); + ++stage_cnt_; + } if (res == 0) { LOG(ERROR) << "read page failed, reach end of file, file id:" @@ -2152,6 +2208,10 @@ void IouringMgr::PollComplete() if (type == UserDataType::KvTaskPageRead) { read_budget_.Release(1, task->IsBackground()); + if (io_stats_enabled_) + { + task->op_cqe_us_ = loop_now_us_; + } } task->io_res_ = cqe->res; task->io_flags_ = cqe->flags; @@ -2245,6 +2305,76 @@ void IouringMgr::PollComplete() io_uring_cq_advance(&ring_, cnt); waiting_sqe_.WakeN(cnt); + assert(inflight_ios_ >= cnt); + inflight_ios_ -= cnt; + + if (io_stats_enabled_) + { + loop_now_us_ = Shard::ReadTimeMicroseconds(); + if (round_prev_us_ != 0) + { + const uint64_t r = loop_now_us_ - round_prev_us_; + round_sum_us_ += r; + round_max_us_ = std::max(round_max_us_, r); + ++round_cnt_; + } + round_prev_us_ = loop_now_us_; + ++stats_iters_; + if (cnt > 0) + { + ++stats_polls_nonzero_; + stats_cqes_ += cnt; + ++stats_batch_hist_[cnt > 8 ? 0 : cnt]; + } + if (loop_now_us_ >= stats_next_flush_us_) + { + if (stats_next_flush_us_ != 0 && VLOG_IS_ON(1) && stats_cqes_ > 0) + { + const uint64_t n = stage_cnt_ > 0 ? stage_cnt_ : 1; + VLOG(1) << "opstages n=" << stage_cnt_ + << " gate_avg=" << stage_sum_us_[0] / n + << " sqe2cqe_avg=" << stage_sum_us_[1] / n + << " cqe2res_avg=" << stage_sum_us_[2] / n + << " q_wait_avg=" << stage_sum_us_[3] / n + << " start_lag_avg=" << stage_sum_us_[5] / n + << " task2gate_avg=" << stage_sum_us_[4] / n + << " q_wait_max=" << stage_max_us_[3] + << " start_lag_max=" << stage_max_us_[5] + << " task2gate_max=" << stage_max_us_[4] + << " round_avg_ns=" + << (round_cnt_ ? round_sum_us_ * 1000 / round_cnt_ : 0) + << " round_max_us=" << round_max_us_ + << " gate_max=" << stage_max_us_[0] + << " sqe2cqe_max=" << stage_max_us_[1] + << " cqe2res_max=" << stage_max_us_[2]; + std::fill( + std::begin(stage_sum_us_), std::end(stage_sum_us_), 0); + std::fill( + std::begin(stage_max_us_), std::end(stage_max_us_), 0); + stage_cnt_ = 0; + round_sum_us_ = 0; + round_max_us_ = 0; + round_cnt_ = 0; + VLOG(1) << "loopstats tid=" << syscall(SYS_gettid) + << " iters/s=" << stats_iters_ / 5 + << " cqes/s=" << stats_cqes_ / 5 << " avg_batch=" + << static_cast(stats_cqes_) / + static_cast(stats_polls_nonzero_) + << " hist(1..8,9+)=" << stats_batch_hist_[1] << "," + << stats_batch_hist_[2] << "," << stats_batch_hist_[3] + << "," << stats_batch_hist_[4] << "," + << stats_batch_hist_[5] << "," << stats_batch_hist_[6] + << "," << stats_batch_hist_[7] << "," + << stats_batch_hist_[8] << "," << stats_batch_hist_[0]; + } + stats_iters_ = 0; + stats_polls_nonzero_ = 0; + stats_cqes_ = 0; + std::fill( + std::begin(stats_batch_hist_), std::end(stats_batch_hist_), 0); + stats_next_flush_us_ = loop_now_us_ + 5'000'000; + } + } } int IouringMgr::MakeDir(FdIdx dir_fd, const char *path) @@ -3293,6 +3423,7 @@ io_uring_sqe *IouringMgr::GetSQE(UserDataType type, const void *user_ptr) // state does not leak to non-fixed operations. sqe->flags = 0; ThdTask()->inflight_io_++; + ++inflight_ios_; prepared_sqe_++; return sqe; } @@ -4395,7 +4526,7 @@ std::pair CloudStoreMgr::TrimRestoredCacheUsage() bool CloudStoreMgr::IsIdle() { - return file_cleaner_.status_ == TaskStatus::Idle && + return IouringMgr::IsIdle() && file_cleaner_.status_ == TaskStatus::Idle && pending_gc_cleanup_.empty() && active_prewarm_tasks_ == 0 && inflight_cloud_slots_ == 0 && !obj_store_.HasPendingWork(); } diff --git a/src/eloq_store.cpp b/src/eloq_store.cpp index d11d3401f..1644cf0c8 100644 --- a/src/eloq_store.cpp +++ b/src/eloq_store.cpp @@ -2196,6 +2196,10 @@ bool EloqStore::SendRequest(KvRequest *req) } req->err_ = KvError::NoError; + if (IoStatsEnabled()) + { + req->dbg_enqueue_us_ = Shard::ReadTimeMicroseconds(); + } #ifdef ELOQ_MODULE_ENABLED { std::lock_guard lk(req->mutex_); diff --git a/src/kv_options.cpp b/src/kv_options.cpp index 944654eae..1f6215fb4 100644 --- a/src/kv_options.cpp +++ b/src/kv_options.cpp @@ -157,7 +157,7 @@ int KvOptions::LoadFromIni(const char *path) if (reader.HasValue(sec_run, "max_inflight_read")) { max_inflight_read = - reader.GetUnsigned(sec_run, "max_inflight_read", 32); + reader.GetUnsigned(sec_run, "max_inflight_read", 64); } if (reader.HasValue(sec_run, "bg_read_ratio")) { diff --git a/src/storage/object_store.cpp b/src/storage/object_store.cpp index 6f6233253..c4f8745db 100644 --- a/src/storage/object_store.cpp +++ b/src/storage/object_store.cpp @@ -793,6 +793,10 @@ bool AsyncHttpManager::SetupUploadRequest(ObjectStore::UploadTask *task, "Content-Length: " + std::to_string(task->file_size_); task->headers_ = curl_slist_append(task->headers_, content_length.c_str()); task->headers_ = curl_slist_append(task->headers_, "Expect:"); + // Suppress libcurl's default form-urlencoded Content-Type. It is not + // covered by the presigned-URL signature (SignedHeaders=host), and strict + // validators (e.g. s3proxy) reject the request when it is present. + task->headers_ = curl_slist_append(task->headers_, "Content-Type:"); // Add conditional headers if provided if (!task->if_match_.empty()) diff --git a/src/storage/shard.cpp b/src/storage/shard.cpp index 266edb6c1..12da75370 100644 --- a/src/storage/shard.cpp +++ b/src/storage/shard.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -180,21 +181,60 @@ void Shard::WorkLoop() } #endif - io_mgr_->Submit(); - - io_mgr_->PollComplete(); - PromoteReadyDelayedReopenRequests(); - ExecuteReadyTasks(); - - int nreqs = dequeue_requests(); - if (nreqs < 0) + if (!IoStatsEnabled()) { - // Exit. - break; + io_mgr_->Submit(); + io_mgr_->PollComplete(); + PromoteReadyDelayedReopenRequests(); + ExecuteReadyTasks(); + int nreqs = dequeue_requests(); + if (nreqs < 0) + { + break; + } + for (int i = 0; i < nreqs; i++) + { + OnReceivedReq(reqs[i]); + } } - for (int i = 0; i < nreqs; i++) + else { - OnReceivedReq(reqs[i]); + // Stats mode: time each loop phase; report any round > 1ms + // with its phase breakdown to locate rare multi-ms stalls. + timespec cpu0; + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &cpu0); + const uint64_t t0 = ReadTimeMicroseconds(); + io_mgr_->Submit(); + const uint64_t t1 = ReadTimeMicroseconds(); + io_mgr_->PollComplete(); + const uint64_t t2 = ReadTimeMicroseconds(); + PromoteReadyDelayedReopenRequests(); + const uint64_t t3 = ReadTimeMicroseconds(); + ExecuteReadyTasks(); + const uint64_t t4 = ReadTimeMicroseconds(); + int nreqs = dequeue_requests(); + if (nreqs < 0) + { + break; + } + for (int i = 0; i < nreqs; i++) + { + OnReceivedReq(reqs[i]); + } + const uint64_t t5 = ReadTimeMicroseconds(); + if (t5 - t0 > 1000) + { + timespec cpu1; + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &cpu1); + const uint64_t cpu_us = + (cpu1.tv_sec - cpu0.tv_sec) * 1000000ULL + + (cpu1.tv_nsec - cpu0.tv_nsec) / 1000; + LOG(INFO) << "SLOWROUND total=" << t5 - t0 + << "us cpu=" << cpu_us << "us submit=" << t1 - t0 + << " poll=" << t2 - t1 << " promote=" << t3 - t2 + << " execute=" << t4 - t3 << " intake=" << t5 - t4 + << " nreqs=" << nreqs; + } } #ifdef ELOQSTORE_WITH_TXSERVICE @@ -563,6 +603,10 @@ GlobalRegisteredMemory *Shard::GlobalRegMem() void Shard::OnReceivedReq(KvRequest *req) { + if (IoStatsEnabled()) + { + req->dbg_dequeue_us_ = ReadTimeMicroseconds(); + } if (req->Reopen()) { req->SetReopen(false); diff --git a/src/tasks/read_task.cpp b/src/tasks/read_task.cpp index 7e5d71535..7530c316e 100644 --- a/src/tasks/read_task.cpp +++ b/src/tasks/read_task.cpp @@ -35,6 +35,10 @@ KvError LocateAndProcess(const TableIdent &tbl_id, uint64_t &expire_ts, Handler &&handler) { + if (IoStatsEnabled()) + { + ThdTask()->op_start_us_ = Shard::ReadTimeMicroseconds(); + } auto [root_handle, err] = shard->IndexManager()->FindRoot(tbl_id); CHECK_KV_ERR(err); RootMeta *meta = root_handle.Get(); From 9e3a8c0156b0db68125fcc1ddc12007f095a39b9 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 16 Jul 2026 06:45:07 +0000 Subject: [PATCH 06/30] fix(io): preserve pending background budget demand --- include/async_io_manager.h | 78 +++++----- include/eloq_store.h | 6 +- include/fail_point.h | 55 +++++-- src/async_io_manager.cpp | 104 +++++++++---- src/eloq_store.cpp | 5 - tests/eloq_store_test.cpp | 4 +- tests/io_qos.cpp | 291 +++++++++++++++++++++++++++++++++++++ 7 files changed, 454 insertions(+), 89 deletions(-) diff --git a/include/async_io_manager.h b/include/async_io_manager.h index 8acfa6b3c..3333374d2 100644 --- a/include/async_io_manager.h +++ b/include/async_io_manager.h @@ -57,9 +57,9 @@ class ManifestFile /** * @brief Per-shard IO QoS statistics (see docs/design/io_qos.md). * - * Snapshot semantics follow TailScratchAcquireCount: counters are mutated - * only by the owning shard thread and read without synchronization by - * tests/diagnostics; values are exact once the shard is quiesced. + * Counters are single-writer relaxed atomics so tests/diagnostics can sample + * a race-free, non-coherent snapshot while the shard is live. Values are exact + * once the shard is quiesced. */ struct IoQosStats { @@ -78,8 +78,9 @@ struct IoQosStats uint64_t admitted_pages_{0}; }; Budget read_; - // Background slice of read_ (M2): pages admitted by background tasks, - // bounded by the bg sub-budget. Always <= read_ component-wise. + // Background slice of read_ (M2), bounded by the bg sub-budget. Inflight, + // high-watermark, and admitted pages are subsets of read_; blocked fields + // are per-class (read_ is foreground, bg_read_ is background). Budget bg_read_; Budget write_; uint64_t fdatasync_count_{0}; // write-path fdatasync ops (FdatasyncFiles) @@ -97,13 +98,14 @@ struct IoQosStats * Optional background sub-budget (M2): when `bg_cap_` is non-zero, * acquisitions with `background = true` are additionally bounded by * `bg_inflight_ <= bg_cap_`. Background never exceeds its slice. Foreground - * may consume the entire budget while background has no queued demand; once - * background waiters exist, their unused entitlement (bg_cap_ - - * bg_inflight_) is reserved and new foreground admissions leave it alone, - * so background always ramps to its share — sustained foreground + * may consume the entire budget while background has no pending demand; once + * background acquisitions enter the wait path, their unused entitlement + * (bg_cap_ - bg_inflight_) is reserved and new foreground admissions leave it + * alone, so background always ramps to its share — sustained foreground * saturation cannot starve it. Each class waits on its own FIFO zone; * release wakes background waiters first (freed units are reserved for - * them while they queue) and forwards unused wake credits to foreground. + * them while they queue) and also wakes foreground so a saturated background + * queue cannot strand foreground waiters. * * A cap of 0 disables the budget (Acquire/Release are no-ops). A request * whose cost exceeds the (sub-)cap (e.g. a merged write larger than a small @@ -126,36 +128,39 @@ class IoBudget void Release(uint32_t cost, bool background = false); IoQosStats::Budget Stats() const { - return {inflight_, - high_watermark_, - blocked_count_, - blocked_us_, - admitted_pages_}; + return {inflight_.load(std::memory_order_relaxed), + high_watermark_.load(std::memory_order_relaxed), + blocked_count_.load(std::memory_order_relaxed), + blocked_us_.load(std::memory_order_relaxed), + admitted_pages_.load(std::memory_order_relaxed)}; } IoQosStats::Budget BgStats() const { - return {bg_inflight_, - bg_high_watermark_, - bg_blocked_count_, - bg_blocked_us_, - bg_admitted_pages_}; + return {bg_inflight_.load(std::memory_order_relaxed), + bg_high_watermark_.load(std::memory_order_relaxed), + bg_blocked_count_.load(std::memory_order_relaxed), + bg_blocked_us_.load(std::memory_order_relaxed), + bg_admitted_pages_.load(std::memory_order_relaxed)}; } private: uint32_t cap_{0}; - uint32_t inflight_{0}; + std::atomic inflight_{0}; uint32_t bg_cap_{0}; // 0 = no background sub-budget - uint32_t bg_inflight_{0}; - // The remaining fields are observability only (tests, tuning, metrics); - // admission decisions read nothing but the caps and inflight counters. - uint32_t high_watermark_{0}; - uint64_t blocked_count_{0}; - uint64_t blocked_us_{0}; - uint64_t admitted_pages_{0}; - uint32_t bg_high_watermark_{0}; - uint64_t bg_blocked_count_{0}; - uint64_t bg_blocked_us_{0}; - uint64_t bg_admitted_pages_{0}; + std::atomic bg_inflight_{0}; + // BG acquisitions that entered the wait path but have not admitted yet. + // A wake does not end demand: the task can yield or re-wait before admit. + uint32_t bg_pending_{0}; + // The remaining atomic fields are observability only (tests, tuning, + // metrics); admission decisions additionally read bg_pending_. + std::atomic high_watermark_{0}; + std::atomic blocked_count_{0}; + std::atomic blocked_us_{0}; + std::atomic admitted_pages_{0}; + std::atomic bg_high_watermark_{0}; + std::atomic bg_blocked_count_{0}; + std::atomic bg_blocked_us_{0}; + std::atomic bg_admitted_pages_{0}; WaitingZone waiting_; // foreground waiters WaitingZone bg_waiting_; // background waiters }; @@ -1172,8 +1177,8 @@ class IouringMgr : public AsyncIoManager IoBudget read_budget_; IoBudget write_budget_; // Write-path fdatasync instrumentation (FdatasyncFiles batches). - uint64_t fdatasync_count_{0}; - uint64_t fdatasync_us_{0}; + std::atomic fdatasync_count_{0}; + std::atomic fdatasync_us_{0}; // Counter for consecutive Submit() calls that skipped the kernel // entry (no prepared SQEs and IORING_SQ_TASKRUN not set). When the @@ -1311,8 +1316,9 @@ class IouringMgr : public AsyncIoManager stats.read_ = read_budget_.Stats(); stats.bg_read_ = read_budget_.BgStats(); stats.write_ = write_budget_.Stats(); - stats.fdatasync_count_ = fdatasync_count_; - stats.fdatasync_us_ = fdatasync_us_; + stats.fdatasync_count_ = + fdatasync_count_.load(std::memory_order_relaxed); + stats.fdatasync_us_ = fdatasync_us_.load(std::memory_order_relaxed); return stats; } diff --git a/include/eloq_store.h b/include/eloq_store.h index 7ce0c34c6..355f41e6e 100644 --- a/include/eloq_store.h +++ b/include/eloq_store.h @@ -986,9 +986,9 @@ class EloqStore /** * @brief Per-shard IO QoS statistics (in-flight page-IO budgets, - * fdatasync accounting; see docs/design/io_qos.md). Counters are - * mutated by the shard thread and read here without synchronization — - * exact once the shard is quiesced, diagnostic-quality otherwise. + * fdatasync accounting; see docs/design/io_qos.md). Counters use + * single-writer relaxed atomics, yielding a race-free but non-coherent + * live snapshot and exact values once the shard is quiesced. * Returns zeros for invalid shard IDs or managers without budgets. */ IoQosStats GetIoQosStats(size_t shard_id) const; diff --git a/include/fail_point.h b/include/fail_point.h index 90008e92f..cabe92efe 100644 --- a/include/fail_point.h +++ b/include/fail_point.h @@ -1,18 +1,24 @@ #pragma once +#include #include // Test-only error injection. Unlike KillPoint (kill_point.h), which SIGTERMs -// the process to exercise crash-recovery paths, FailPoint makes a code path -// return an error so tests can drive in-process error/abort handling (e.g. -// verifying WriteTask::Abort rolls back the BranchFileMapping high-water -// marks). +// the process to exercise crash-recovery paths, FailPoint perturbs a code path +// in-process (for example, returning an error or forcing a scheduler yield). // -// Usage: arm a named point from the test, then a TEST_FAIL_POINT_RETURN(name, -// err) embedded in engine code returns `err` exactly once before -// auto-disarming. Compiled out entirely in release (NDEBUG) builds, like the -// kill-point macros. +// Usage: ArmOnce auto-disarms after a matching TEST_FAIL_POINT_RETURN or +// TEST_FAIL_POINT_ACTION; ArmPersistent fires every match until Disarm. +// Compiled out entirely in release (NDEBUG) builds, like the kill-point macros. #ifndef NDEBUG +#define TEST_FAIL_POINT_ACTION(name, action) \ + do \ + { \ + if (::eloqstore::FailPoint::GetInstance().ShouldFail(name)) \ + { \ + action; \ + } \ + } while (0) #define TEST_FAIL_POINT_RETURN(name, err) \ do \ { \ @@ -22,6 +28,10 @@ } \ } while (0) #else +#define TEST_FAIL_POINT_ACTION(name, action) \ + do \ + { \ + } while (0) #define TEST_FAIL_POINT_RETURN(name, err) \ do \ { \ @@ -45,25 +55,42 @@ class FailPoint // @p name must be a string literal (stored by pointer, not copied). void ArmOnce(const char *name) { - armed_ = name; + persistent_.store(false, std::memory_order_relaxed); + armed_.store(name, std::memory_order_release); + } + + // Arm until Disarm. Used by scheduler regressions that must perturb every + // matching wake throughout a bounded observation window. + void ArmPersistent(const char *name) + { + persistent_.store(true, std::memory_order_relaxed); + armed_.store(name, std::memory_order_release); } void Disarm() { - armed_ = nullptr; + armed_.store(nullptr, std::memory_order_release); } bool ShouldFail(const char *name) { - if (armed_ == nullptr || std::strcmp(armed_, name) != 0) + const char *armed = armed_.load(std::memory_order_acquire); + if (armed == nullptr || std::strcmp(armed, name) != 0) { return false; } - armed_ = nullptr; // fire once - return true; + if (persistent_.load(std::memory_order_relaxed)) + { + return true; + } + return armed_.compare_exchange_strong(armed, + nullptr, + std::memory_order_acq_rel, + std::memory_order_acquire); } private: - const char *armed_{nullptr}; + std::atomic armed_{nullptr}; + std::atomic persistent_{false}; }; } // namespace eloqstore diff --git a/src/async_io_manager.cpp b/src/async_io_manager.cpp index 45c7957dd..d16788652 100644 --- a/src/async_io_manager.cpp +++ b/src/async_io_manager.cpp @@ -178,56 +178,93 @@ void IoBudget::Acquire(uint32_t cost, bool background) // escape per class: a request with cost > (sub-)cap is admitted alone // once the relevant count drains, guaranteeing progress (see IoBudget // doc comment). Background is additionally bounded by its sub-budget. - // Foreground must not take units reserved for *queued* background - // demand (bg_cap_ - bg_inflight_ while bg_waiting_ is non-empty): - // without that reservation, sustained foreground saturation would + // Foreground must not take units reserved for pending background demand. + // Pending spans the full first-wait-to-admission interval, including a + // wake followed by a yield or re-wait, when the wait queue is empty. + // Without that reservation, sustained foreground saturation would // starve background forever — every freed unit would be re-acquired by // a foreground waiter and compaction could never bootstrap its share. - // While background has no queued demand, foreground may use the entire + // While background has no pending demand, foreground may use the entire // budget. auto must_wait = [this, cost, use_bg]() { + const uint32_t inflight = inflight_.load(std::memory_order_relaxed); if (use_bg) { - if (inflight_ + cost > cap_ && inflight_ != 0) + if (inflight + cost > cap_ && inflight != 0) { return true; } - return bg_inflight_ + cost > bg_cap_ && bg_inflight_ != 0; - } - const uint32_t reserved = - (bg_cap_ != 0 && !bg_waiting_.Empty()) ? bg_cap_ - bg_inflight_ : 0; - return inflight_ + cost > cap_ - reserved && inflight_ != 0; + const uint32_t bg_inflight = + bg_inflight_.load(std::memory_order_relaxed); + return bg_inflight + cost > bg_cap_ && bg_inflight != 0; + } + const uint32_t bg_inflight = + bg_inflight_.load(std::memory_order_relaxed); + const uint32_t reserved = bg_pending_ != 0 && bg_inflight < bg_cap_ + ? bg_cap_ - bg_inflight + : 0; + const uint32_t fg_cap = cap_ > reserved ? cap_ - reserved : 0; + return inflight + cost > fg_cap && inflight != 0; }; // Each class queues behind its own existing waiters (approximate FIFO // across arrivals within the class). WaitingZone &zone = use_bg ? bg_waiting_ : waiting_; + bool pending_bg = false; if (!zone.Empty() || must_wait()) { // TSC-based clock (see Shard::ReadTimeMicroseconds): Acquire always // runs on the shard thread, after Shard::Init calibrated the TSC. const uint64_t start_us = shard->ReadTimeMicroseconds(); - (use_bg ? bg_blocked_count_ : blocked_count_)++; + std::atomic &blocked_count = + use_bg ? bg_blocked_count_ : blocked_count_; + blocked_count.store(blocked_count.load(std::memory_order_relaxed) + 1, + std::memory_order_relaxed); + if (use_bg) + { + ++bg_pending_; + pending_bg = true; + } do { zone.Wait(ThdTask()); + if (use_bg) + { + TEST_FAIL_POINT_ACTION("IoBudgetBgWake", + ThdTask()->YieldToLowPQ()); + } } while (must_wait()); const uint64_t waited_us = shard->DurationMicroseconds(start_us); - (use_bg ? bg_blocked_us_ : blocked_us_) += waited_us; + std::atomic &blocked_us = + use_bg ? bg_blocked_us_ : blocked_us_; + blocked_us.store(blocked_us.load(std::memory_order_relaxed) + waited_us, + std::memory_order_relaxed); + } + if (pending_bg) + { + CHECK_GT(bg_pending_, 0); + --bg_pending_; } - inflight_ += cost; - admitted_pages_ += cost; - if (inflight_ > high_watermark_) + const uint32_t inflight = inflight_.load(std::memory_order_relaxed) + cost; + inflight_.store(inflight, std::memory_order_relaxed); + admitted_pages_.store( + admitted_pages_.load(std::memory_order_relaxed) + cost, + std::memory_order_relaxed); + if (inflight > high_watermark_.load(std::memory_order_relaxed)) { - high_watermark_ = inflight_; + high_watermark_.store(inflight, std::memory_order_relaxed); } if (use_bg) { - bg_inflight_ += cost; - bg_admitted_pages_ += cost; - if (bg_inflight_ > bg_high_watermark_) + const uint32_t bg_inflight = + bg_inflight_.load(std::memory_order_relaxed) + cost; + bg_inflight_.store(bg_inflight, std::memory_order_relaxed); + bg_admitted_pages_.store( + bg_admitted_pages_.load(std::memory_order_relaxed) + cost, + std::memory_order_relaxed); + if (bg_inflight > bg_high_watermark_.load(std::memory_order_relaxed)) { - bg_high_watermark_ = bg_inflight_; + bg_high_watermark_.store(bg_inflight, std::memory_order_relaxed); } } } @@ -238,12 +275,15 @@ void IoBudget::Release(uint32_t cost, bool background) { return; } - assert(inflight_ >= cost); - inflight_ -= cost; + const uint32_t inflight = inflight_.load(std::memory_order_relaxed); + CHECK_GE(inflight, cost); + inflight_.store(inflight - cost, std::memory_order_relaxed); if (background && bg_cap_ != 0) { - assert(bg_inflight_ >= cost); - bg_inflight_ -= cost; + const uint32_t bg_inflight = + bg_inflight_.load(std::memory_order_relaxed); + CHECK_GE(bg_inflight, cost); + bg_inflight_.store(bg_inflight - cost, std::memory_order_relaxed); } // Each freed page-unit can admit at most one waiter; over-waking is safe // because woken tasks re-check the admission condition and re-wait. @@ -253,7 +293,7 @@ void IoBudget::Release(uint32_t cost, bool background) // would be futile. Unused wake credits are forwarded to foreground — // when background is saturated or idle, all credits go to foreground. size_t woken = 0; - if (bg_cap_ != 0 && bg_inflight_ < bg_cap_) + if (bg_cap_ != 0 && bg_inflight_.load(std::memory_order_relaxed) < bg_cap_) { woken = bg_waiting_.WakeN(cost); } @@ -1470,7 +1510,7 @@ KvError IouringMgr::SubmitMergedWrite(const TableIdent &tbl_id, // Write-budget admission (io_qos.md M1): cost in 4KB-page units so the // cap means the same thing in append and non-append mode. Must mirror // the release cost computed from bytes_ in PollComplete. - write_budget_.Acquire(MergedWriteCost(bytes)); + write_budget_.Acquire(MergedWriteCost(req->bytes_)); io_uring_sqe *sqe = GetSQE(UserDataType::MergedWriteReq, req); auto [fd, registered] = req->fd_ref_.FdPair(); if (registered) @@ -2232,6 +2272,7 @@ void IouringMgr::PollComplete() case UserDataType::WriteReq: { WriteReq *req = static_cast(ptr); + TEST_FAIL_POINT_ACTION("WriteReqCqe", cqe->res = -EIO); KvError err; assert(cqe->res <= options_->data_page_size); if (cqe->res < 0) @@ -2260,6 +2301,7 @@ void IouringMgr::PollComplete() case UserDataType::MergedWriteReq: { MergedWriteReq *req = static_cast(ptr); + TEST_FAIL_POINT_ACTION("MergedWriteReqCqe", cqe->res = -EIO); KvError err; if (cqe->res < 0) { @@ -2305,7 +2347,7 @@ void IouringMgr::PollComplete() io_uring_cq_advance(&ring_, cnt); waiting_sqe_.WakeN(cnt); - assert(inflight_ios_ >= cnt); + CHECK_GE(inflight_ios_, cnt); inflight_ios_ -= cnt; if (io_stats_enabled_) @@ -2570,8 +2612,12 @@ KvError IouringMgr::FdatasyncFiles(const TableIdent &tbl_id, io_uring_prep_fsync(sqe, fd, IORING_FSYNC_DATASYNC); } ThdTask()->WaitIo(); - fdatasync_count_ += reqs.size(); - fdatasync_us_ += shard->DurationMicroseconds(fsync_start_us); + fdatasync_count_.store( + fdatasync_count_.load(std::memory_order_relaxed) + reqs.size(), + std::memory_order_relaxed); + fdatasync_us_.store(fdatasync_us_.load(std::memory_order_relaxed) + + shard->DurationMicroseconds(fsync_start_us), + std::memory_order_relaxed); // Check results. KvError err = KvError::NoError; diff --git a/src/eloq_store.cpp b/src/eloq_store.cpp index 1644cf0c8..1487741d4 100644 --- a/src/eloq_store.cpp +++ b/src/eloq_store.cpp @@ -197,11 +197,6 @@ bool EloqStore::ValidateOptions(KvOptions &opts) LOG(ERROR) << "Invalid option overflow_pointers"; return false; } - if (opts.max_write_batch_pages == 0) - { - LOG(ERROR) << "Invalid option max_write_batch_pages"; - return false; - } if (!opts.cloud_store_path.empty()) { LOG(ERROR) << "cloud mode already support standby, reset " diff --git a/tests/eloq_store_test.cpp b/tests/eloq_store_test.cpp index 391143a4b..6466a149d 100644 --- a/tests/eloq_store_test.cpp +++ b/tests/eloq_store_test.cpp @@ -64,9 +64,9 @@ TEST_CASE("EloqStore ValidateOptions validates all parameters", "[eloq_store]") REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == false); options = CreateValidOptions(test_dir); // restore valid value - // Test invalid max_write_batch_pages + // The retired per-task write throttle no longer constrains validation. options.max_write_batch_pages = 0; - REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == false); + REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == true); options = CreateValidOptions(test_dir); // restore valid value // Test invalid max_cloud_concurrency (cloud mode) diff --git a/tests/io_qos.cpp b/tests/io_qos.cpp index 9c025c85c..5c1e406eb 100644 --- a/tests/io_qos.cpp +++ b/tests/io_qos.cpp @@ -11,6 +11,7 @@ #include "async_io_manager.h" #include "common.h" +#include "fail_point.h" #include "test_utils.h" using test_util::MapVerifier; @@ -215,6 +216,71 @@ TEST_CASE("io budgets: failed write drains the budget", "[io_qos]") recover.Read(7); } +TEST_CASE("io budgets: negative WriteReq CQE drains and recovers", "[io_qos]") +{ + eloqstore::KvOptions opts = default_opts; + opts.max_inflight_write = 8; + eloqstore::EloqStore *store = InitStore(opts); + const eloqstore::TableIdent tbl_id{"qos-write-cqe", 0}; + const uint64_t ts = utils::UnixTs(); + + eloqstore::BatchWriteRequest failed; + failed.SetTableId(tbl_id); + for (uint32_t i = 0; i < 100; ++i) + { + failed.AddWrite(test_util::Key(i, 7), + std::string(200, 'w'), + ts, + eloqstore::WriteOp::Upsert); + } + eloqstore::FailPoint::GetInstance().ArmOnce("WriteReqCqe"); + store->ExecSync(&failed); + eloqstore::FailPoint::GetInstance().Disarm(); + + REQUIRE(failed.Error() != eloqstore::KvError::NoError); + REQUIRE(ShardStats(store).write_.inflight_ == 0); + + eloqstore::BatchWriteRequest recovery; + recovery.SetTableId(tbl_id); + recovery.AddWrite("recovery", "value", ts + 1, eloqstore::WriteOp::Upsert); + store->ExecSync(&recovery); + REQUIRE(recovery.Error() == eloqstore::KvError::NoError); + REQUIRE(ShardStats(store).write_.inflight_ == 0); +} + +TEST_CASE("io budgets: negative MergedWriteReq CQE drains and recovers", + "[io_qos]") +{ + eloqstore::KvOptions opts = append_opts; + opts.max_inflight_write = 64; + eloqstore::EloqStore *store = InitStore(opts); + const eloqstore::TableIdent tbl_id{"qos-merged-cqe", 0}; + const uint64_t ts = utils::UnixTs(); + + eloqstore::BatchWriteRequest failed; + failed.SetTableId(tbl_id); + for (uint32_t i = 0; i < 400; ++i) + { + failed.AddWrite(test_util::Key(i, 7), + std::string(3000, 'm'), + ts, + eloqstore::WriteOp::Upsert); + } + eloqstore::FailPoint::GetInstance().ArmOnce("MergedWriteReqCqe"); + store->ExecSync(&failed); + eloqstore::FailPoint::GetInstance().Disarm(); + + REQUIRE(failed.Error() != eloqstore::KvError::NoError); + REQUIRE(ShardStats(store).write_.inflight_ == 0); + + eloqstore::BatchWriteRequest recovery; + recovery.SetTableId(tbl_id); + recovery.AddWrite("recovery", "value", ts + 1, eloqstore::WriteOp::Upsert); + store->ExecSync(&recovery); + REQUIRE(recovery.Error() == eloqstore::KvError::NoError); + REQUIRE(ShardStats(store).write_.inflight_ == 0); +} + TEST_CASE("io budgets: shutdown while tasks queue behind the budget", "[io_qos]") { @@ -377,6 +443,231 @@ TEST_CASE("bg sub-budget: batch-write leaf loads are background", "[io_qos]") REQUIRE(stats.bg_read_.high_watermark_ <= 2); } +TEST_CASE("bg sub-budget: pending demand survives the wake gap", "[io_qos]") +{ + eloqstore::KvOptions opts = default_opts; + opts.num_threads = 1; + opts.max_inflight_read = 2; + opts.bg_read_ratio = 50; // bg cap = 1 + opts.overflow_pointers = 128; + eloqstore::EloqStore *store = InitStore(opts); + + const eloqstore::TableIdent fg_tbl{"qos-wake-fg", 0}; + MapVerifier fg_seed(fg_tbl, store, false); + fg_seed.SetValueSize(600 * 1024); + fg_seed.Upsert(0); + + const eloqstore::TableIdent bg_tbl{"qos-wake-bg", 0}; + MapVerifier bg_seed(bg_tbl, store, false); + bg_seed.SetValueSize(200); + bg_seed.Upsert(0, 2000); + + std::atomic stop_fg{false}; + std::atomic fg_failed{false}; + std::atomic fg_done{0}; + eloqstore::FailPoint::GetInstance().ArmPersistent("IoBudgetBgWake"); + + std::vector readers; + readers.reserve(8); + for (int i = 0; i < 8; ++i) + { + readers.emplace_back( + [&] + { + while (!stop_fg.load(std::memory_order_relaxed)) + { + eloqstore::ReadRequest req; + req.SetArgs(fg_tbl, test_util::Key(0, 7)); + store->ExecSync(&req); + if (req.Error() != eloqstore::KvError::NoError) + { + fg_failed.store(true, std::memory_order_relaxed); + } + fg_done.fetch_add(1, std::memory_order_relaxed); + } + }); + } + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + + eloqstore::BatchWriteRequest bg_req; + bg_req.SetTableId(bg_tbl); + bg_req.AddWrite(test_util::Key(1000, 7), + std::string(200, 'b'), + utils::UnixTs() + 1, + eloqstore::WriteOp::Upsert); + std::atomic bg_done{false}; + const uint64_t fg_before = fg_done.load(std::memory_order_relaxed); + store->ExecAsyn(&bg_req, + 0, + [&bg_done](eloqstore::KvRequest *) + { bg_done.store(true, std::memory_order_relaxed); }); + + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::milliseconds(200); + while (std::chrono::steady_clock::now() < deadline) + { + if (bg_done.load(std::memory_order_relaxed) && + fg_done.load(std::memory_order_relaxed) > fg_before) + { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + const bool bg_completed_while_armed = + bg_done.load(std::memory_order_relaxed); + const uint64_t fg_after = fg_done.load(std::memory_order_relaxed); + + stop_fg.store(true, std::memory_order_relaxed); + eloqstore::FailPoint::GetInstance().Disarm(); + for (std::thread &reader : readers) + { + reader.join(); + } + store->Stop(); + + const eloqstore::IoQosStats stats = ShardStats(store); + CAPTURE(fg_before, fg_after); + REQUIRE(fg_after > fg_before); + REQUIRE(bg_completed_while_armed); + REQUIRE_FALSE(fg_failed.load(std::memory_order_relaxed)); + REQUIRE(bg_done.load(std::memory_order_relaxed)); + REQUIRE(bg_req.Error() == eloqstore::KvError::NoError); + REQUIRE(stats.read_.inflight_ == 0); + REQUIRE(stats.bg_read_.inflight_ == 0); + REQUIRE(stats.read_.high_watermark_ <= 2); + REQUIRE(stats.bg_read_.high_watermark_ <= 1); + REQUIRE(stats.bg_read_.blocked_count_ > 0); +} + +TEST_CASE("bg sub-budget: repeated ungated contention", "[io_qos][stress]") +{ + eloqstore::KvOptions opts = default_opts; + opts.num_threads = 1; + opts.max_inflight_read = 2; + opts.bg_read_ratio = 50; + opts.overflow_pointers = 128; + eloqstore::EloqStore *store = InitStore(opts); + + const eloqstore::TableIdent fg_tbl{"qos-stress-fg", 0}; + MapVerifier fg_seed(fg_tbl, store, false); + fg_seed.SetValueSize(600 * 1024); + fg_seed.Upsert(0); + + const eloqstore::TableIdent bg_tbl{"qos-stress-bg", 0}; + MapVerifier bg_seed(bg_tbl, store, false); + bg_seed.SetValueSize(200); + bg_seed.Upsert(0, 2000); + + std::atomic failed{false}; + for (uint64_t round = 1; round <= 10; ++round) + { + std::atomic ready{0}; + std::atomic start{false}; + std::vector readers; + readers.reserve(8); + for (int i = 0; i < 8; ++i) + { + readers.emplace_back( + [&] + { + ready.fetch_add(1, std::memory_order_relaxed); + while (!start.load(std::memory_order_relaxed)) + { + std::this_thread::yield(); + } + for (int read = 0; read < 4; ++read) + { + eloqstore::ReadRequest req; + req.SetArgs(fg_tbl, test_util::Key(0, 7)); + store->ExecSync(&req); + if (req.Error() != eloqstore::KvError::NoError) + { + failed.store(true, std::memory_order_relaxed); + } + } + }); + } + while (ready.load(std::memory_order_relaxed) != readers.size()) + { + std::this_thread::yield(); + } + start.store(true, std::memory_order_relaxed); + + eloqstore::BatchWriteRequest bg_req; + bg_req.SetTableId(bg_tbl); + bg_req.AddWrite(test_util::Key(1000, 7), + std::string(200, 'b'), + utils::UnixTs() + round, + eloqstore::WriteOp::Upsert); + store->ExecSync(&bg_req); + if (bg_req.Error() != eloqstore::KvError::NoError) + { + failed.store(true, std::memory_order_relaxed); + } + for (std::thread &reader : readers) + { + reader.join(); + } + } + + const eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE_FALSE(failed.load(std::memory_order_relaxed)); + REQUIRE(stats.read_.inflight_ == 0); + REQUIRE(stats.bg_read_.inflight_ == 0); + REQUIRE(stats.read_.high_watermark_ <= 2); + REQUIRE(stats.bg_read_.high_watermark_ <= 1); + REQUIRE(stats.bg_read_.blocked_count_ > 0); +} + +TEST_CASE("io qos stats: concurrent sampling", "[io_qos][stats]") +{ + eloqstore::KvOptions opts = default_opts; + opts.max_inflight_read = 2; + opts.max_inflight_write = 8; + eloqstore::EloqStore *store = InitStore(opts); + + std::atomic stop{false}; + std::atomic samples{0}; + std::atomic workload_active{false}; + std::atomic active_samples{0}; + std::thread sampler( + [&] + { + while (!stop.load(std::memory_order_relaxed)) + { + const eloqstore::IoQosStats stats = store->GetIoQosStats(0); + (void) stats; + samples.fetch_add(1, std::memory_order_relaxed); + if (workload_active.load(std::memory_order_relaxed)) + { + active_samples.fetch_add(1, std::memory_order_relaxed); + } + } + }); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(3000); + workload_active.store(true, std::memory_order_relaxed); + for (int round = 0; round < 3; ++round) + { + verify.WriteRnd(0, 1000, 0, 50); + for (int i = 0; i < 100; ++i) + { + verify.Read(std::rand() % 1000); + } + } + workload_active.store(false, std::memory_order_relaxed); + + stop.store(true, std::memory_order_relaxed); + sampler.join(); + const eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(samples.load(std::memory_order_relaxed) > 0); + REQUIRE(active_samples.load(std::memory_order_relaxed) > 0); + REQUIRE(stats.read_.inflight_ == 0); + REQUIRE(stats.bg_read_.inflight_ == 0); + REQUIRE(stats.write_.inflight_ == 0); +} + TEST_CASE("io budgets: defaults are behavior-neutral", "[io_qos]") { // At default caps (read 32, write 32768) a single-threaded unit From e5259cc8cdd3192617382967498211c61eb2c6bd Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 16 Jul 2026 07:27:27 +0000 Subject: [PATCH 07/30] fix(benchmark): harden GET2 lifecycle and timing --- benchmark/eloq_store_bm.cc | 120 +++++++++++++++++++++++++++---------- include/async_io_manager.h | 4 +- src/async_io_manager.cpp | 5 +- src/storage/shard.cpp | 26 +++++--- 4 files changed, 114 insertions(+), 41 deletions(-) diff --git a/benchmark/eloq_store_bm.cc b/benchmark/eloq_store_bm.cc index 2923ffb72..0657d810a 100644 --- a/benchmark/eloq_store_bm.cc +++ b/benchmark/eloq_store_bm.cc @@ -549,7 +549,9 @@ struct Get2Client { moodycamel::BlockingConcurrentQueue done_; std::vector lat_us_; - uint64_t completed_{0}; + uint64_t outstanding_{0}; + uint64_t successes_{0}; + uint64_t read_failed_{0}; uint64_t issue_failed_{0}; }; @@ -571,7 +573,26 @@ void Benchmark::RunGet2(uint32_t client_threads, uint32_t inflight, uint32_t per_shard_cap) { - const uint32_t nshards = worker_cnt_ > 0 ? worker_cnt_ : 1; + CHECK_GT(client_threads, 0U) << "GET2 client_threads must be positive"; + CHECK_GT(inflight, 0U) << "GET2 inflight_per_client must be positive"; + CHECK_GT(partition_count_, 0U) << "GET2 partition_count must be positive"; + + const uint16_t nshards = worker_cnt_; + std::vector partition_shards(partition_count_); + std::vector reachable_shards(nshards, false); + for (uint32_t part = 0; part < partition_count_; ++part) + { + const ::eloqstore::TableIdent table_id(table_name_str, part); + partition_shards[part] = table_id.ShardIndex(nshards); + reachable_shards[partition_shards[part]] = true; + } + const uint64_t reachable_count = + std::count(reachable_shards.begin(), reachable_shards.end(), true); + const uint64_t cap_capacity = reachable_count * per_shard_cap; + CHECK(per_shard_cap == 0 || inflight <= cap_capacity) + << "GET2 inflight_per_client=" << inflight + << " exceeds reachable per-shard capacity=" << cap_capacity; + std::atomic stop{false}; std::vector clients(client_threads); std::vector thds; @@ -580,7 +601,14 @@ void Benchmark::RunGet2(uint32_t client_threads, for (uint32_t c = 0; c < client_threads; ++c) { thds.emplace_back( - [this, c, inflight, per_shard_cap, nshards, &clients, &stop]() + [this, + c, + inflight, + per_shard_cap, + nshards, + &partition_shards, + &clients, + &stop]() { Get2Client &me = clients[c]; object_generator gen; @@ -599,7 +627,6 @@ void Benchmark::RunGet2(uint32_t client_threads, ops.back().client_ = &me; } std::vector shard_out(nshards, 0); - uint64_t issued = 0; auto issue = [&](ReadOperation *op) -> bool { @@ -608,17 +635,25 @@ void Benchmark::RunGet2(uint32_t client_threads, uint32_t part = key_index % partition_count_; if (per_shard_cap > 0) { - for (int tries = 0; - shard_out[part % nshards] >= per_shard_cap && - tries < 8; - ++tries) + uint32_t selected = partition_count_; + for (uint32_t offset = 0; offset < partition_count_; + ++offset) { - key_index = - gen.get_key_index(OBJECT_GENERATOR_KEY_RANDOM); - part = key_index % partition_count_; + const uint32_t candidate = + (static_cast(part) + offset) % + partition_count_; + if (shard_out[partition_shards[candidate]] < + per_shard_cap) + { + selected = candidate; + break; + } } + CHECK_LT(selected, partition_count_) + << "GET2 per-shard cap accounting lost capacity"; + part = selected; } - op->shard_ = part % nshards; + op->shard_ = partition_shards[part]; op->key_.clear(); gen.generate_key(key_index, op->key_); op->req_->SetArgs( @@ -633,38 +668,53 @@ void Benchmark::RunGet2(uint32_t client_threads, return false; } ++shard_out[op->shard_]; - ++issued; + ++me.outstanding_; return true; }; + auto complete = [&](ReadOperation *op) + { + CHECK_GT(me.outstanding_, 0U); + CHECK_GT(shard_out[op->shard_], 0U); + --me.outstanding_; + --shard_out[op->shard_]; + if (op->req_->Error() != ::eloqstore::KvError::NoError) + { + ++me.read_failed_; + return; + } + me.lat_us_.push_back(Get2NowUs() - op->start_ts_); + ++me.successes_; + }; + for (auto &op : ops) { + if (stop.load(std::memory_order_acquire)) + { + break; + } issue(&op); } ReadOperation *done_op = nullptr; - while (!stop.load(std::memory_order_relaxed)) + while (!stop.load(std::memory_order_acquire)) { if (!me.done_.wait_dequeue_timed(done_op, 10000)) { continue; } - const uint64_t now = Get2NowUs(); - me.lat_us_.push_back(now - done_op->start_ts_); - ++me.completed_; - --shard_out[done_op->shard_]; - issue(done_op); - } - // Drain remaining in-flight before exiting. - uint64_t drained = me.completed_; - const uint64_t deadline = Get2NowUs() + 3000000; - while (drained + me.issue_failed_ < issued && - Get2NowUs() < deadline) - { - if (me.done_.wait_dequeue_timed(done_op, 10000)) + complete(done_op); + if (!stop.load(std::memory_order_acquire)) { - ++drained; + issue(done_op); } } + // The callbacks reference `ops` and `me`; keep both alive + // until every accepted request has completed. + while (me.outstanding_ > 0) + { + me.done_.wait_dequeue(done_op); + complete(done_op); + } }); } @@ -677,10 +727,14 @@ void Benchmark::RunGet2(uint32_t client_threads, const double dur_sec = (Get2NowUs() - bench_start) / 1e6; std::vector all; - uint64_t total = 0; + uint64_t successes = 0; + uint64_t read_failures = 0; + uint64_t issue_failures = 0; for (auto &cl : clients) { - total += cl.completed_; + successes += cl.successes_; + read_failures += cl.read_failed_; + issue_failures += cl.issue_failed_; all.insert(all.end(), cl.lat_us_.begin(), cl.lat_us_.end()); } std::sort(all.begin(), all.end()); @@ -696,9 +750,11 @@ void Benchmark::RunGet2(uint32_t client_threads, }; LOG(INFO) << "GET2 finished: clients=" << client_threads << " inflight=" << inflight << " per_shard_cap=" << per_shard_cap - << " completed=" << total << " duration=" << dur_sec + << " successes=" << successes + << " read_failures=" << read_failures + << " issue_failures=" << issue_failures << " duration=" << dur_sec << "s QPS:" << std::fixed << std::setprecision(2) - << total / dur_sec; + << successes / dur_sec; LOG(INFO) << "Latency: Min->" << (all.empty() ? 0 : all.front()) << ", Max->" << (all.empty() ? 0 : all.back()) << ", Mean->" << (all.empty() ? 0 diff --git a/include/async_io_manager.h b/include/async_io_manager.h index 3333374d2..f44b00bbf 100644 --- a/include/async_io_manager.h +++ b/include/async_io_manager.h @@ -1201,7 +1201,9 @@ class IouringMgr : public AsyncIoManager // flushed as one VLOG(1) line every 5s: iteration rate, CQE rate, // and the reap-batch-size histogram (how many CQEs each non-empty // PollComplete found). Used to observe completion-batching regimes. - uint64_t loop_now_us_{0}; // stamped by Submit() each loop iteration + // Stamped once at PollComplete entry and shared by that round's CQEs and + // loop statistics. + uint64_t loop_now_us_{0}; uint64_t stats_next_flush_us_{0}; uint64_t stats_iters_{0}; uint64_t stats_polls_nonzero_{0}; diff --git a/src/async_io_manager.cpp b/src/async_io_manager.cpp index d16788652..00c27feb9 100644 --- a/src/async_io_manager.cpp +++ b/src/async_io_manager.cpp @@ -2230,6 +2230,10 @@ void IouringMgr::Submit() void IouringMgr::PollComplete() { + if (io_stats_enabled_) + { + loop_now_us_ = Shard::ReadTimeMicroseconds(); + } io_uring_cqe *cqe = nullptr; io_uring_peek_cqe(&ring_, &cqe); unsigned head; @@ -2352,7 +2356,6 @@ void IouringMgr::PollComplete() if (io_stats_enabled_) { - loop_now_us_ = Shard::ReadTimeMicroseconds(); if (round_prev_us_ != 0) { const uint64_t r = loop_now_us_ - round_prev_us_; diff --git a/src/storage/shard.cpp b/src/storage/shard.cpp index 12da75370..ec578640a 100644 --- a/src/storage/shard.cpp +++ b/src/storage/shard.cpp @@ -131,7 +131,8 @@ void Shard::WorkLoop() // and no active tasks. // This allows the thread to exit gracefully when the store is stopped. std::array reqs; - auto dequeue_requests = [this, &reqs]() -> int + auto dequeue_requests = [this, + &reqs](uint64_t *queue_wait_us = nullptr) -> int { size_t nreqs = requests_.try_dequeue_bulk(reqs.data(), reqs.size()); // Idle state, wait for new requests or exit. @@ -151,8 +152,14 @@ void Shard::WorkLoop() return 0; } const auto timeout = std::chrono::milliseconds(100); + const uint64_t wait_start = + queue_wait_us == nullptr ? 0 : ReadTimeMicroseconds(); nreqs = requests_.wait_dequeue_bulk_timed( reqs.data(), reqs.size(), timeout); + if (queue_wait_us != nullptr) + { + *queue_wait_us += ReadTimeMicroseconds() - wait_start; + } } return nreqs; @@ -212,7 +219,8 @@ void Shard::WorkLoop() const uint64_t t3 = ReadTimeMicroseconds(); ExecuteReadyTasks(); const uint64_t t4 = ReadTimeMicroseconds(); - int nreqs = dequeue_requests(); + uint64_t queue_wait_us = 0; + int nreqs = dequeue_requests(&queue_wait_us); if (nreqs < 0) { break; @@ -222,17 +230,21 @@ void Shard::WorkLoop() OnReceivedReq(reqs[i]); } const uint64_t t5 = ReadTimeMicroseconds(); - if (t5 - t0 > 1000) + const uint64_t total_us = t5 - t0; + const uint64_t active_us = total_us - queue_wait_us; + if (active_us > 1000) { timespec cpu1; clock_gettime(CLOCK_THREAD_CPUTIME_ID, &cpu1); const uint64_t cpu_us = (cpu1.tv_sec - cpu0.tv_sec) * 1000000ULL + (cpu1.tv_nsec - cpu0.tv_nsec) / 1000; - LOG(INFO) << "SLOWROUND total=" << t5 - t0 - << "us cpu=" << cpu_us << "us submit=" << t1 - t0 - << " poll=" << t2 - t1 << " promote=" << t3 - t2 - << " execute=" << t4 - t3 << " intake=" << t5 - t4 + LOG(INFO) << "SLOWROUND total=" << total_us + << "us active=" << active_us << "us cpu=" << cpu_us + << "us submit=" << t1 - t0 << " poll=" << t2 - t1 + << " promote=" << t3 - t2 << " execute=" << t4 - t3 + << " intake=" << t5 - t4 - queue_wait_us + << " queue_wait=" << queue_wait_us << " nreqs=" << nreqs; } } From ee53563d04066fd615b6bd04d255233fbf4c8e4c Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 16 Jul 2026 08:04:32 +0000 Subject: [PATCH 08/30] fix(io): wire QoS metrics and align docs --- CLAUDE.md | 2 + benchmark/interference_bench.cpp | 4 +- benchmark/opts_interference.ini | 2 +- docs/architecture/04-execution-model.md | 12 +-- docs/architecture/07-io-stack.md | 29 ++++--- docs/design/io_qos.md | 107 +++++++++++++----------- docs/design/io_qos_impl_plan.md | 65 +++++++------- include/async_io_manager.h | 22 +++-- include/kv_options.h | 23 +++-- src/async_io_manager.cpp | 15 ++-- src/eloq_store.cpp | 7 ++ tests/io_qos.cpp | 6 +- 12 files changed, 162 insertions(+), 132 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e96105104..dc505300b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,6 +71,8 @@ Full subsystem documentation lives in `docs/architecture/` (see above). The load **I/O stack is layered, not parallel implementations** (`include/async_io_manager.h`): `IouringMgr` is the local-filesystem base (page buffers, manifests, FD cache, direct I/O, merged writes); `CloudStoreMgr` extends it, treating local files as a cache over object storage (`ObjectStore` → `CloudStorageService` → `AsyncHttpManager`); `StandbyStoreMgr` extends it filling missing state via rsync jobs in `StandbyService`. Local manifest/page semantics are the common substrate in every mode — remote sync and cache cleanup must preserve local replayability. +**Page-I/O budget symmetry:** acquire after every other blocking resource, do not yield between admission and SQE submission, and release the identical cost exactly once per CQE. + **On-disk and in-memory page formats intentionally differ** (`src/storage/`). `DataPageBuilder`/`IndexPageBuilder` define the canonical encoded layout (prefix-compressed keys, restart points, timestamp deltas, overflow flags); large values go to chained overflow pages. In memory, internal nodes are swizzled `MemIndexPage` objects with pins and cached child pointers, managed by `IndexPageManager`. Any file-format change must update both builder and reader paths. **Durability is anchored at RootMeta + manifest, not pages** (`src/storage/root_meta.cpp`, `src/storage/page_mapper.cpp`, `src/replayer.cpp`). Logical-to-file page identity is versioned through `MappingSnapshot`s for COW root transitions; `PageMapper` owns the logical page table and allocator, reconstructed from manifest replay. `Replayer` handles both cold restart and live refresh (reopen, cloud/standby snapshot install) — manifest encoding changes must keep both working. Swizzled pages can only be recycled after being unswizzled from every live mapping snapshot. diff --git a/benchmark/interference_bench.cpp b/benchmark/interference_bench.cpp index e0e583f31..69d573d10 100644 --- a/benchmark/interference_bench.cpp +++ b/benchmark/interference_bench.cpp @@ -65,8 +65,8 @@ DEFINE_uint32(val_size, DEFINE_uint32(read_concurrency, 32, "concurrent point reads (closed loop)"); DEFINE_uint32(baseline_secs, 15, "seconds of read-only baseline phase"); DEFINE_uint32(storm_secs, 60, "seconds of read + write-storm phase"); -DEFINE_uint32(storm_ratio, 5, "overwrite span out of every ratio keys"); -DEFINE_uint32(storm_span, 3, "overwrite span out of every ratio keys"); +DEFINE_uint32(storm_ratio, 5, "key-stride width of the overwrite pattern"); +DEFINE_uint32(storm_span, 3, "keys overwritten within each stride"); DEFINE_uint32(storm_batch_keys, 2048, "keys per storm batch-write request"); DEFINE_uint32(write_read_ratio, 9, diff --git a/benchmark/opts_interference.ini b/benchmark/opts_interference.ini index 5669fcfb5..1a5ab4097 100644 --- a/benchmark/opts_interference.ini +++ b/benchmark/opts_interference.ini @@ -19,7 +19,7 @@ num_retained_archives = 0 skip_verify_checksum = true # --- IO QoS knobs under test --- -max_inflight_read = 32 +max_inflight_read = 64 bg_read_ratio = 25 max_inflight_write = 512 diff --git a/docs/architecture/04-execution-model.md b/docs/architecture/04-execution-model.md index 93134b6f9..94b3f703f 100644 --- a/docs/architecture/04-execution-model.md +++ b/docs/architecture/04-execution-model.md @@ -66,11 +66,13 @@ Scheduling primitives: so release never depends on the blocked task being scheduled. Background tasks (`KvTask::IsBackground()`: BatchWrite, BackgroundWrite, EvictFile, Prewarm) are additionally confined to a read sub-budget (`bg_read_ratio`) - and wait on a separate FIFO zone. While background waiters queue, their - unused sub-budget is reserved from new foreground admissions (neither - class can starve the other); release wakes background first and forwards - unused wake credits to foreground. See `docs/design/io_qos.md` (M1/M2); - the acquire order is FD/mutex → pools/buffers → budget → SQE. + and wait on a separate FIFO zone. From a background acquisition's first wait + through admission (including the wake-to-admit gap), its unused sub-budget is + reserved from new foreground admissions. Release wakes background first and + always wakes foreground; both classes re-check admission, so neither can + starve the other. See `docs/design/io_qos.md` (M1/M2); the acquire order is + FD/mutex → pools/buffers → budget → SQE, with no voluntary yield after budget + admission and an equal-cost release per CQE. `TaskManager` keeps one free-list pool per task type (`BatchWriteTask`, `BackgroundWrite`, `ReadTask`, `ScanTask`, `ListObjectTask`, diff --git a/docs/architecture/07-io-stack.md b/docs/architecture/07-io-stack.md index df7903700..9dcd88a24 100644 --- a/docs/architecture/07-io-stack.md +++ b/docs/architecture/07-io-stack.md @@ -12,7 +12,7 @@ Source: `include/async_io_manager.h`, `src/async_io_manager.cpp`, `AsyncIoManager` is the shard-facing storage interface. One instance per shard, chosen by `AsyncIoManager::Instance` from the store mode: -``` +```text AsyncIoManager (abstract: ReadPage/WritePage/ReadSegments/Manifest ops/...) ├── MemStoreMgr store_path empty — pages and manifests in RAM (tests) └── IouringMgr local filesystem over io_uring — the substrate @@ -38,7 +38,8 @@ Responsibilities: reads when the buffer is registered), `WritePage`. `ConvFilePageId` splits a `FilePageId` into `(file_id, offset)` by `pages_per_file_shift`. - **In-flight page-IO budgets** (`IoBudget`, see `docs/design/io_qos.md` - M1/M2) — two per-shard counters in 4KB-page units with independent caps: + M1/M2) — two per-shard counters in configured `data_page_size` units with + independent caps: `read_budget_` (`max_inflight_read`; `ReadPage`/`ReadPages`, per-page acquisition so a batch larger than the cap cannot deadlock) and `write_budget_` (`max_inflight_write`; `WritePage` cost 1, @@ -47,21 +48,25 @@ Responsibilities: distinguishes budgeted page reads from metadata ops via the `KvTaskPageRead`/`BaseReqPageRead` user-data types. A cap of 0 disables a budget; a single request costlier than the cap is admitted alone once the - budget drains. Metadata, manifest, and segment IO are exempt. + budget drains. Metadata, manifest, and segment IO are exempt. With + `enable_data_page_cache`, `max_inflight_write` also bounds cached-page pins + retained by write promotion until the corresponding IO completes. The read budget carries a **background sub-budget** (`bg_read_ratio` percent of `max_inflight_read`): page reads from tasks where `KvTask::IsBackground()` (BatchWrite, BackgroundWrite, EvictFile, Prewarm) are additionally bounded by it, so compaction/GC/batch-write read bursts - cannot crowd foreground point reads out of the device queue. Foreground - may use the entire read budget while background has no queued demand; - once background waiters exist their unused sub-budget is reserved from - new foreground admissions, so neither class can starve the other. Each - class waits on its own FIFO zone; release wakes background first and - forwards unused wake credits to foreground. The write budget has no - split — all page writes come from write tasks, i.e. background. + cannot crowd foreground point reads out of the device queue. Foreground may + use the entire read budget while background has no pending demand. Once a + background acquisition enters the wait path, its unused sub-budget stays + reserved through admission, including the wake-to-admit gap. Each class + waits on its own FIFO zone; release wakes background first and always wakes + foreground. The write budget has no split — all page writes come from write + tasks, i.e. background. `GetIoQosStats()` (also surfaced as `EloqStore::GetIoQosStats(shard_id)`) - exposes in-flight/high-watermark/blocked counters (read, bg-read slice, - write) plus write-path fdatasync count and latency. + exposes in-flight/high-watermark/blocked counters (total read, bg-read slice, + write) plus write-path fdatasync count and latency. The blocked fields are + per admission class: `read_` counts foreground waits, `bg_read_` background + waits, and `write_` all write waits. - **FD cache** — `LruFD` per (partition, `TypedFileId`), doubly-linked LRU bounded by the shard's fd budget; `EvictFD` closes idle descriptors. Open/close exclusion per FD via a coroutine `Mutex`. Data files open with diff --git a/docs/design/io_qos.md b/docs/design/io_qos.md index 9b2a39c3b..9e4ae0108 100644 --- a/docs/design/io_qos.md +++ b/docs/design/io_qos.md @@ -14,15 +14,16 @@ time-budgeted cooperative yield (`MaybeYield()` / `eloqstore_yield_budget_us`, default 20µs) bounds how long any single compaction/GC coroutine segment holds the worker thread, and GC unlink/close SQE preparation is chunked into 128-op batches. What remains — and what this -document addresses — is that **CPU priority is lost at the IO boundary**. -Within one 20µs slice a background task can still burst up to 128 page reads -(compaction move batches, `DoCompactDataFile`) plus writes into the io_uring -ring, and nothing distinguishes those requests from foreground reads at the -ring or device level. Reads are not budgeted at all; writes are budgeted only -per-task. Yielding controls when background IO is *issued*, not how much of -it queues at the device once issued. +document addresses — was that **CPU priority was lost at the IO boundary**. +Before M1/M2, one 20µs slice could still burst up to 128 page reads (compaction +move batches, `DoCompactDataFile`) plus writes into the io_uring ring, and +nothing distinguished those requests from foreground reads at the ring or +device level. Reads were not budgeted at all; writes were budgeted only +per-task. Yielding controlled when background IO was *issued*, not how much of +it queued at the device once issued. -This document proposes three per-shard mechanisms: +This document defines three per-shard mechanisms. M1 and M2 are implemented; +M3 remains a measurement-gated follow-up: - **M1**: per-shard caps on in-flight page IO, with **separate caps for reads and writes** — they are different device resources and must be @@ -43,11 +44,14 @@ page/segment IO, pools), `08-data-lifecycle.md` (compaction/GC cadence). Implementing this design requires updating those docs in the same change (see CLAUDE.md). -## Current Mechanisms and Gaps +## Baseline and Current Mechanisms -### What exists today +### Pre-QoS baseline -| Mechanism | Location | Scope | +The table and gaps below describe the baseline before M1/M2, not the current +implementation. + +| Baseline mechanism | Location | Scope | |---|---|---| | Two ready queues (high / low priority), 400µs low-priority slice | `Shard::ExecuteReadyTasks`, `KvTask::YieldToLowPQ` | CPU only | | Time-budgeted cooperative yield: `MaybeYield()` once a coroutine segment exceeds `eloqstore_yield_budget_us` (20µs) — added by #455 | `MaybeYield`, `Shard::CurResumeElapsedUs` | CPU only | @@ -59,22 +63,28 @@ Implementing this design requires updating those docs in the same change | `max_write_concurrency` bounds concurrent write tasks (incl. compaction: `GetBackgroundWrite` returns null at the limit) | `TaskManager` | Default unlimited in local mode | | Cloud slots (`max_cloud_concurrency`), cloud buffer pool | `CloudStoreMgr` | Cloud HTTP, not disk | -### Gaps +### Baseline gaps addressed by M1/M2 -1. **Reads have no budget anywhere.** Page compaction +1. **Reads had no budget anywhere.** Page compaction (`DoCompactDataFile`) issues bursts of up to `max_read_pages_batch` (128) page reads per move batch through the same `ReadPages` path as foreground reads. -2. **No shard-global in-flight IO counter.** Only per-task `inflight_io_` and - `prepared_sqe_` exist. The effective global bounds (4096 SQEs, 32K +2. **There was no shard-global in-flight IO counter.** Only per-task + `inflight_io_` and `prepared_sqe_` existed. The effective global bounds + (4096 SQEs, 32K in-flight writes) are far beyond the queue depth at which NVMe read latency degrades. -3. **No FG/BG tagging of IO.** SQEs are indistinguishable once submitted; - `sqe->ioprio` is never set. A background task scheduled for one 20µs +3. **There was no FG/BG tagging of IO.** SQEs were indistinguishable once + submitted; `sqe->ioprio` was never set. A background task scheduled for one 20µs slice can leave 128+ requests queued at the device ahead of every subsequent foreground read — the yield discipline of #455 bounds CPU occupation per slice, not the IO submitted within it. +The current M1/M2 implementation uses per-shard defaults of 64 configured +data pages for reads, a 25% background read slice, and 512 configured data +pages for writes. The old `max_write_batch_pages` throttle has no effect; M3 +is still deferred. + ## Device-Level Rationale EloqStore's pattern in append mode is **sequential writes vs. random reads**, @@ -107,8 +117,9 @@ Consequences for the design: ### M1: Per-shard in-flight page-IO caps, reads and writes separate -Two per-shard counters of in-flight **page** IO (in 4KB-page units), with -independent caps, enforced at the page-IO entry points of `IouringMgr`. +Two per-shard counters of in-flight **page** IO (in configured +`data_page_size` units), with independent caps, enforced at the page-IO entry +points of `IouringMgr`. Reads and writes are deliberately **not** mixed in one budget: they are different device resources (reads need deep queue depth for IOPS; writes saturate bandwidth at shallow depth and their interference tracks sustained @@ -119,9 +130,9 @@ separately. `ReadPage` / `ReadPages`, cost 1 per page. - Write budget (`inflight_write_pages_` ≤ `max_inflight_write`, option redefined — see Interaction with Existing Knobs): - `WritePage`, cost 1; `SubmitMergedWrite`, cost = bytes / data_page_size - (a 1MB merged write counts as 256 pages), so the cap means the same thing - in append and non-append mode. + `WritePage`, cost 1; `SubmitMergedWrite`, cost = bytes rounded up to + `data_page_size` (a 1MB merged write counts as 256 pages at the default 4KB + page size), so the cap means the same thing in append and non-append mode. Segment IO (`ReadSegments` / `WriteSegments`, zero-copy large values) is **out of scope** — see Non-Goals. @@ -134,9 +145,9 @@ in-flight counter holds only previously admitted, not-yet-completed IO; each request is counted exactly once, from admission to completion. For a new request of size `cost` (page-units, not yet counted): -``` +```text acquire(class, cost): // at the page-IO entry point - while inflight[class] + cost > cap[class]: + while inflight[class] != 0 and inflight[class] + cost > cap[class]: wait on waiting_zone[class] (FIFO) inflight[class] += cost // admitted; prep SQE(s) and submit @@ -184,9 +195,10 @@ Read budget structure: `max_inflight_read`, e.g. 25%). Background never exceeds its sub-budget. Foreground can consume the entire -read budget **while background has no queued demand**; once background -waiters exist, their unused entitlement (`bg_read_limit − bg_inflight`) is -reserved — new foreground admissions leave it alone. Without the +read budget **while background has no pending demand**; once a background +acquisition enters the wait path, its unused entitlement +(`bg_read_limit − bg_inflight`) stays reserved through admission, including +the wake-to-admit gap — new foreground admissions leave it alone. Without the reservation, sustained foreground saturation would starve background forever (every freed unit would be re-acquired by a foreground waiter and compaction could never bootstrap its share), letting space amplification @@ -229,22 +241,20 @@ compaction, since interference tracks bytes/sec rather than queue depth. ### New options (per shard) -``` -uint32_t max_inflight_read = 64; // pages; 0 = disabled +```cpp +uint32_t max_inflight_read = 64; // configured data pages; 0 = disabled uint32_t bg_read_ratio = 25; // percent of max_inflight_read -uint32_t max_inflight_write = 512; // pages; REDEFINED existing option +uint32_t max_inflight_write = 512; // configured data pages; redefined option // (was 32768, pool sizing only) uint64_t bg_write_rate_limit = 0; // bytes/sec; 0 = disabled (M3) ``` -bg_read_ratio = 25 was set from the WSL interference sweeps (2026-07-03): -total caps 32–256 were indistinguishable at fixed bg_cap, 32 matched that -box's BDP and gave the campaign's best p50/p999, and bg_cap 8 (25% of 32) -sat exactly at the measured background demand line — tail protection at zero write-throughput cost (the -write-throughput knee appeared only at bg_cap 4). Real-device calibration -(the QD sweep below) should re-derive max_inflight_read per device; the -ratio is policy and should transfer. max_inflight_write's real default -still awaits commit 4. +The WSL interference sweeps (2026-07-03) initially selected a read cap of 32 +and `bg_read_ratio = 25`. Azure NVMe validation (2026-07-11) then showed real +foreground queueing at 32 while 64–128 were indistinguishable, so the shipped +read default is 64 and the policy ratio remains 25%. The shipped write default +is 512. Real-device calibration (the QD sweep below) should still re-derive the +read cap per device; the ratio is policy and should transfer. ### Sizing contract: device knob × policy knob @@ -304,8 +314,8 @@ The two read-side options deliberately live at different levels: `max_inflight_write` rather than by the per-task drain, so pathologically small buffer pools must account for in-flight-write pins (production pools dwarf the ≤ max_inflight_write pages of pins; only tests noticed). -- **`max_inflight_write` is redefined, not retired.** It becomes the M1 - write cap (page units, default dropped from 32768 to a few hundred). +- **`max_inflight_write` is redefined, not retired.** It is the M1 write cap + (configured data-page units, default dropped from 32768 to 512). `WriteReqPool` stays sized to it; in-flight writes can never exceed it, so the pool bound and the QoS bound coincide. Note this is a behavioral change for deployments that set the old option explicitly. @@ -344,17 +354,16 @@ All budgets are per shard, but the device is shared by all shards waiters — no cross-thread coroutine resumption needed. - Cloud mode: NIC bandwidth is shared between background uploads and foreground cache-miss downloads; the same FG/BG discipline should - eventually apply to cloud slots. Also note upload-path disk reads - (`ReadFilePrefix`) are issued by background write tasks and are therefore - automatically counted against the BG disk budget — remember this when - sizing it in cloud mode. + eventually apply to cloud slots. Upload-path bulk disk reads + (`ReadFilePrefix`) remain exempt from the page-IO budget. ## Non-Goals - **Segment IO (zero-copy large values) is not budgeted.** Segments exist to serve very large values, a workload where throughput matters rather than tail latency and fairness, so FG/BG isolation is not a goal there. - Segment IO is neither counted against `max_inflight_io` nor classified; + Segment IO is counted against neither `max_inflight_read` nor + `max_inflight_write`, and is not classified; segment compaction keeps its existing bounds (`max_segments_batch` in-flight segments per batch via the shared `GlobalRegisteredMemory` pool, `segment_compact_yield_every`). Revisit only if large-value traffic is @@ -370,8 +379,10 @@ All budgets are per shard, but the device is shared by all shards Export per-shard counters from day one; tuning must be measurement-driven: -- Current and high-watermark in-flight pages, split FG/BG, read/write. -- Cumulative blocked-time and block counts per class (budget wait). +- Current and high-watermark pages for total reads, the BG-read subset, and + writes; foreground read usage is total minus BG. +- Cumulative blocked time and counts per admission class: `read_` is foreground + waits only, `bg_read_` is background waits, and `write_` is all write waits. - Background write bytes/sec (actual, vs. limit when M3 lands). - fdatasync count and latency histogram. @@ -395,7 +406,7 @@ Export per-shard counters from day one; tuning must be measurement-driven: 3. **Verify TRIM**: confirm the filesystem issues discards for GC-deleted files (`discard` mount option or periodic `fstrim`); otherwise the WAF ≈ 1 assumption silently breaks on aged drives. -4. **Staged rollout**: +4. **Historical staged rollout** (M1/M2 and throttle retirement are complete): - Land M1+M2 with existing throttles left in place (loosened). - Re-run the read-vs-compaction benchmark; confirm the new mechanism alone protects foreground p99. diff --git a/docs/design/io_qos_impl_plan.md b/docs/design/io_qos_impl_plan.md index be084d627..74dc7cdce 100644 --- a/docs/design/io_qos_impl_plan.md +++ b/docs/design/io_qos_impl_plan.md @@ -59,14 +59,15 @@ ample room. Current types: `KvTask`, `BaseReq`, `WriteReq`, `MergedWriteReq`. Add: - `KvTaskPageRead` — single-page read issued via the `KvTask` path - (`IouringMgr::ReadPage`). Cost 1. Handled identically to `KvTask` in + (`IouringMgr::ReadPage`). Cost 1 configured data page. Handled identically to + `KvTask` in `PollComplete`, plus `read_budget_.Release(1)`. - `BaseReqPageRead` — per-page read in a batch (`IouringMgr::ReadPages`). Handled identically to `BaseReq`, plus `read_budget_.Release(1)`. Metadata ops keep `KvTask`/`BaseReq` and are never charged. Write costs are -already derivable: `WriteReq` = 1 page; `MergedWriteReq` = `bytes_ / -data_page_size` (round up; assert page alignment). +already derivable in configured data-page units: `WriteReq` = 1; +`MergedWriteReq` = `bytes_ / data_page_size` (round up; assert page alignment). FG/BG at release time (needed by commit 2) comes from the task pointer every req type already carries: `task->IsBackground()`. @@ -95,7 +96,7 @@ never by the blocked task) and unreachable in practice with caps ≪ | `IouringMgr::ReadPage` (`async_io_manager.cpp` ~484) | `read_budget_.Acquire(1)` at the top of the `read_page` lambda's retry loop, before `GetSQE`. Release is per-CQE, so each retry re-acquires. | | `IouringMgr::ReadPages` (~565) | Acquire **per page** inside `send_req`, not per batch. Rationale: a 128-page compaction batch must not deadlock against a BG sub-budget of 64 (commit 2); per-page acquisition lets the task block mid-batch while already-submitted pages complete. This supersedes the "atomic batch acquire" idea discussed during design review. | | `IouringMgr::WritePage` (~742) | `write_budget_.Acquire(1)` after `write_req_pool_->Alloc`, before `GetSQE`. | -| `IouringMgr::SubmitMergedWrite` (~777) | `write_budget_.Acquire(bytes / page_size)` after `merged_write_req_pool_->Alloc`, before `GetSQE`. | +| `IouringMgr::SubmitMergedWrite` (~777) | `write_budget_.Acquire(bytes / data_page_size)` after `merged_write_req_pool_->Alloc`, before `GetSQE`. | Exempt (unchanged): all metadata ops, manifest IO, `ReadFile` / `WriteSnapshot` bulk paths, `Fdatasync` (instrumented only), @@ -109,10 +110,10 @@ policy in commit 1 is a plain `WakeN` on the released budget's zone. ### Options and validation -- `kv_options.h`: add `max_inflight_read = 256`; redefine - `max_inflight_write` (page units, keep old default 32768 in this commit — - the default drops in commit 4). Add INI parsing in `kv_options.cpp` and - equality-operator entries. +- `kv_options.h`: add `max_inflight_read` (initially 256; shipped default 64); + redefine `max_inflight_write` in configured data-page units (keep old default + 32768 in this commit — the shipped default drops to 512 in commit 4). Add INI + parsing in `kv_options.cpp` and equality-operator entries. - `eloq_store.cpp` option validation: `max_inflight_write != 0` already enforced; nothing else hard-fails. `max_inflight_read == 0` disables the read budget (documented). @@ -122,10 +123,12 @@ policy in commit 1 is a plain `WakeN` on the released budget's zone. ### Stats `struct IoQosStats` (per budget: current, high-watermark, blocked count, -cumulative blocked µs) + `IouringMgr::GetIoQosStats()`. Wire into the -`ELOQSTORE_WITH_TXSERVICE` metrics meter behind the existing -`EnableMetrics()` guard; otherwise reachable from tests via the store's -shard accessors. Add an fdatasync counter + latency accumulator in +cumulative blocked µs) + `IouringMgr::GetIoQosStats()`. `read_` blocked fields +count foreground waits only, `bg_read_` counts background waits, and `write_` +counts all write waits. The `ELOQSTORE_WITH_TXSERVICE` meter registers gauges +for current total-read, BG-read, and write usage behind the existing +`EnableMetrics()` guard; full stats remain reachable through the store's shard +accessors. Add an fdatasync counter + latency accumulator in `SyncFiles`/`FdatasyncFiles` in the same commit (evaluation step 2 needs it). @@ -147,13 +150,10 @@ it). > taking a `background` flag) rather than as a separate class, so the write > budget reuses the same type with no sub-budget configured; per-class > oversized-request escape and FIFO zones as described; stats gained a -> `bg_read_` slice and a BG-read inflight gauge. (b) The wake policy below -> ("wake FG waiters first") was found to starve background under sustained -> foreground saturation — recycling alone cannot even bootstrap BG's share -> from zero. Replaced by a demand-gated reservation: while BG waiters -> queue, their unused entitlement is off-limits to new FG admissions; -> release wakes BG first and forwards unused wake credits to FG -> (`WaitingZone::WakeN` now returns the count actually woken). +> `bg_read_` slice and a BG-read inflight gauge. (b) Waking FG first was found +> to starve background under sustained foreground saturation. Demand is now +> tracked from first wait through admission, so the unused BG entitlement stays +> reserved across wake-to-admit; release wakes BG first and always wakes FG. - `KvTask::IsBackground()` in `tasks/task.h`: `BatchWrite`, `BackgroundWrite`, `EvictFile`, `Prewarm` → true; explicitly not @@ -162,9 +162,9 @@ it). (`bg_waiting_`). Acquire for a BG task additionally checks `bg_inflight_ + cost > bg_cap_`. Release decrements both counters when the completing request's `task->IsBackground()`. -- Wake policy: on read-budget release, wake FG waiters first (`waiting_`), - then BG waiters only while `bg_inflight_ < bg_cap_` and total headroom - remains. Spurious wakes are safe (acquire re-checks in its while loop). +- Wake policy: on read-budget release, wake BG waiters while + `bg_inflight_ < bg_cap_`, and always wake FG as well. Spurious wakes are safe + (acquire re-checks in its loop). - Option `bg_read_ratio = 25` (percent, clamp 1–100); `bg_cap_ = max_inflight_read * ratio / 100`, min 1 page... but see next line: `bg_cap_` must be ≥ 1 and the per-page acquisition from commit 1 @@ -204,7 +204,8 @@ it). > campaign (real-device confirmation still advisable before release). > Details: `WritePage` throttle branch removed (budget admission comment > left in its place); `max_write_batch_pages` deprecated — parsed with a -> LOG(WARNING), validation kept for compat, field documented as no-effect; +> LOG(WARNING), validation removed so all values are accepted, field documented +> as no-effect; > `max_inflight_write` default 32768 → 512, validated by a {512, 2048, > 32768} sweep (512 binds marginally — hwm pinned, 6–26 blocks/30s — at > equal-or-best write throughput and read tails; natural demand ceiling was @@ -212,19 +213,19 @@ it). > enable_data_page_cache, write-promotion pins are now bounded by > max_inflight_write instead of the per-task drain; the data_page_cache OOM > test was recalibrated to `max_inflight_write = 1` (tiny 2-slot pool). -> Earlier same-day change under this commit's umbrella: -> max_inflight_read default 256 → 32, bg_read_ratio stays 25. +> The read default moved from 256 to 32 in the initial WSL campaign, then to +> the shipped value 64 after Azure NVMe validation; bg_read_ratio stays 25. -Only after the interference benchmark confirms M1+M2 alone protect -foreground p99 (design evaluation step 4): +The implemented change was gated on the interference benchmark confirming +that M1+M2 alone protect foreground p99 (design evaluation step 4): -- Remove the `inflight_io_ >= max_write_batch_pages → WaitWrite()` branch in +- Removed the `inflight_io_ >= max_write_batch_pages → WaitWrite()` branch in `WriteTask::WritePage` (`write_task.cpp` ~304). Keep the CPU yields and the terminal `WaitWrite()`. -- Drop `max_inflight_write` default 32768 → calibrated value (~512). +- Dropped `max_inflight_write` default 32768 → calibrated value 512. Release-notes entry: behavioral change for deployments setting it explicitly. -- Deprecate `max_write_batch_pages` in `kv_options.h` (keep parsing, +- Deprecated `max_write_batch_pages` in `kv_options.h` (keep parsing, ignore, log a warning) for one release before deletion. ## Commit 5 — M3 rate limiter (deferred) @@ -238,8 +239,8 @@ degradation that inflight caps don't remove: token bucket (bytes) in ## Testing plan -> **Status (2026-07-03):** unit-test items 1–6 all implemented in -> `tests/io_qos.cpp` (11 cases). Notes: item 4's "two concurrent merged +> **Status:** unit-test items 1–6 and the later regression cases are implemented +> in `tests/io_qos.cpp` (16 cases). Notes: item 4's "two concurrent merged > writes" requires multiple partitions' write tasks — a single task's > flushes don't overlap (it yields per page while filling the next buffer); > item 5 uses the read-only-directory injection (kill points SIGTERM the diff --git a/include/async_io_manager.h b/include/async_io_manager.h index f44b00bbf..12844a834 100644 --- a/include/async_io_manager.h +++ b/include/async_io_manager.h @@ -90,22 +90,20 @@ struct IoQosStats /** * @brief Per-shard in-flight page-IO budget (M1/M2 in docs/design/io_qos.md). * - * Counts admitted, not-yet-completed page IO in 4KB-page units. Tasks block - * in Acquire when admission would exceed the cap; IouringMgr::PollComplete - * releases per CQE and wakes waiters, so release never depends on the - * blocked task being scheduled. + * Counts admitted, not-yet-completed page IO in configured data-page units + * (`KvOptions::data_page_size`). Tasks block in Acquire when admission would + * exceed the cap; IouringMgr::PollComplete releases per CQE and wakes waiters, + * so release never depends on the blocked task being scheduled. * * Optional background sub-budget (M2): when `bg_cap_` is non-zero, * acquisitions with `background = true` are additionally bounded by * `bg_inflight_ <= bg_cap_`. Background never exceeds its slice. Foreground * may consume the entire budget while background has no pending demand; once - * background acquisitions enter the wait path, their unused entitlement - * (bg_cap_ - bg_inflight_) is reserved and new foreground admissions leave it - * alone, so background always ramps to its share — sustained foreground - * saturation cannot starve it. Each class waits on its own FIFO zone; - * release wakes background waiters first (freed units are reserved for - * them while they queue) and also wakes foreground so a saturated background - * queue cannot strand foreground waiters. + * a background acquisition enters the wait path, its unused entitlement + * (bg_cap_ - bg_inflight_) stays reserved through admission, including the + * wake-to-admit gap. Each class waits on its own FIFO zone; release wakes + * background waiters first and always wakes foreground, so neither sustained + * foreground saturation nor a saturated background queue can starve a class. * * A cap of 0 disables the budget (Acquire/Release are no-ops). A request * whose cost exceeds the (sub-)cap (e.g. a merged write larger than a small @@ -1325,7 +1323,7 @@ class IouringMgr : public AsyncIoManager } /** - * @brief Write-budget cost of a merged write in 4KB-page units. + * @brief Write-budget cost of a merged write in configured data-page units. * Acquire (SubmitMergedWrite) and release (PollComplete) must use this * same formula so the budget balances exactly. */ diff --git a/include/kv_options.h b/include/kv_options.h index e252bc6e9..698ddc23f 100644 --- a/include/kv_options.h +++ b/include/kv_options.h @@ -73,10 +73,12 @@ struct KvOptions */ uint32_t io_queue_size = 4096; /** - * @brief Per-shard cap on in-flight page-write IO, in 4KB-page units - * (docs/design/io_qos.md M1). A merged append-mode write of N bytes - * counts as N / data_page_size pages, so the cap means the same thing - * in append and non-append mode. Also sizes the write request pools. + * @brief Per-shard cap on in-flight page-write IO, in configured + * data-page units (`data_page_size`; docs/design/io_qos.md M1). A merged + * append-mode write is charged by its byte length rounded up to a data + * page, so the cap means the same thing in append and non-append mode. + * Also sizes the write request pools. With `enable_data_page_cache`, it + * bounds cached-page write-promotion pins held until IO completion. * Cannot be zero. * * NOTE: before the IO QoS work this option only sized the non-append @@ -86,8 +88,9 @@ struct KvOptions */ uint32_t max_inflight_write = 512; /** - * @brief Per-shard cap on in-flight page-read IO, in 4KB-page units - * (docs/design/io_qos.md M1). Applies to data-page reads + * @brief Per-shard cap on in-flight page-read IO, in configured + * data-page units (`data_page_size`; docs/design/io_qos.md M1). Applies to + * data-page reads * (ReadPage/ReadPages); metadata and segment IO are exempt. * 0 disables the read budget. * @@ -104,8 +107,10 @@ struct KvOptions * 1..100; docs/design/io_qos.md M2). Page reads issued by background * tasks (batch write, compaction, GC, prewarm) are bounded by this * sub-budget so they cannot crowd out foreground point reads. - * Foreground reads may use the entire read budget. No effect when the - * read budget is disabled (max_inflight_read = 0). + * Foreground reads may use the entire read budget while no background + * acquisition is pending; pending background demand reserves its unused + * share through admission. No effect when the read budget is disabled + * (max_inflight_read = 0). */ uint32_t bg_read_ratio = 25; /** @@ -114,7 +119,7 @@ struct KvOptions * many writes outstanding). Superseded by the shard-wide write budget * `max_inflight_write` (docs/design/io_qos.md, plan commit 4), which * bounds in-flight write pages across all tasks without the - * drain-to-zero sawtooth. Parsed and validated for compatibility; + * drain-to-zero sawtooth. Parsed for compatibility and otherwise ignored; * scheduled for removal one release after deprecation. */ uint32_t max_write_batch_pages = 32; diff --git a/src/async_io_manager.cpp b/src/async_io_manager.cpp index 00c27feb9..8c9ad2889 100644 --- a/src/async_io_manager.cpp +++ b/src/async_io_manager.cpp @@ -287,11 +287,10 @@ void IoBudget::Release(uint32_t cost, bool background) } // Each freed page-unit can admit at most one waiter; over-waking is safe // because woken tasks re-check the admission condition and re-wait. - // Background waiters are woken first: while background has queued - // demand and sub-budget room, freed units are reserved for it (see the - // admission rule in Acquire), so waking foreground for those units - // would be futile. Unused wake credits are forwarded to foreground — - // when background is saturated or idle, all credits go to foreground. + // Background waiters are woken first while the sub-budget has room. + // Their demand and unused entitlement stay reserved through admission, + // including the wake-to-admit gap (see Acquire), so foreground tasks woken + // for those units simply re-check and re-wait. size_t woken = 0; if (bg_cap_ != 0 && bg_inflight_.load(std::memory_order_relaxed) < bg_cap_) { @@ -1507,9 +1506,9 @@ KvError IouringMgr::SubmitMergedWrite(const TableIdent &tbl_id, static_cast(req->pages_.size() - 1); } - // Write-budget admission (io_qos.md M1): cost in 4KB-page units so the - // cap means the same thing in append and non-append mode. Must mirror - // the release cost computed from bytes_ in PollComplete. + // Write-budget admission (io_qos.md M1): cost in configured data-page + // units so the cap means the same thing in append and non-append mode. + // Must mirror the release cost computed from bytes_ in PollComplete. write_budget_.Acquire(MergedWriteCost(req->bytes_)); io_uring_sqe *sqe = GetSQE(UserDataType::MergedWriteReq, req); auto [fd, registered] = req->fd_ref_.FdPair(); diff --git a/src/eloq_store.cpp b/src/eloq_store.cpp index 1487741d4..7a03a5def 100644 --- a/src/eloq_store.cpp +++ b/src/eloq_store.cpp @@ -2371,6 +2371,13 @@ void EloqStore::InitializeMetrics(metrics::MetricsRegistry *metrics_registry, metrics::Type::Gauge); metrics_meters_[i]->Register(metrics::NAME_ELOQSTORE_LOCAL_SPACE_LIMIT, metrics::Type::Gauge); + metrics_meters_[i]->Register( + metrics::NAME_ELOQSTORE_INFLIGHT_READ_PAGES, metrics::Type::Gauge); + metrics_meters_[i]->Register( + metrics::NAME_ELOQSTORE_INFLIGHT_BG_READ_PAGES, + metrics::Type::Gauge); + metrics_meters_[i]->Register( + metrics::NAME_ELOQSTORE_INFLIGHT_WRITE_PAGES, metrics::Type::Gauge); } enable_eloqstore_metrics_ = true; diff --git a/tests/io_qos.cpp b/tests/io_qos.cpp index 5c1e406eb..3f21fff47 100644 --- a/tests/io_qos.cpp +++ b/tests/io_qos.cpp @@ -339,7 +339,7 @@ TEST_CASE("io budgets: disabled read budget stays untouched", "[io_qos]") REQUIRE(stats.read_.inflight_ == 0); REQUIRE(stats.read_.high_watermark_ == 0); REQUIRE(stats.read_.blocked_count_ == 0); - // The write budget (default 32768) still counts, it just never blocks. + // The write budget (default 512) still counts, it just never blocks. REQUIRE(stats.write_.inflight_ == 0); REQUIRE(stats.write_.blocked_count_ == 0); } @@ -670,10 +670,10 @@ TEST_CASE("io qos stats: concurrent sampling", "[io_qos][stats]") TEST_CASE("io budgets: defaults are behavior-neutral", "[io_qos]") { - // At default caps (read 32, write 32768) a single-threaded unit + // At default caps (read 64, write 512) a single-threaded unit // workload of small values must never block on a budget: foreground // point reads are sequential (in-flight 1) and batch-write leaf loads - // are sequential background singles, well under bg_cap = 8. + // are sequential background singles, well under bg_cap = 16. eloqstore::EloqStore *store = InitStore(default_opts); MapVerifier verify(test_tbl_id, store, false); From aa963c5d5186ca782700daf1602d3c95de75f09b Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 16 Jul 2026 08:24:45 +0000 Subject: [PATCH 09/30] fix(benchmark): scale QoS traffic by page size --- benchmark/interference_bench.cpp | 34 +++++++++++++++++++++----------- docs/architecture/07-io-stack.md | 7 ++++--- docs/design/io_qos.md | 9 +++++---- docs/design/io_qos_impl_plan.md | 2 +- include/async_io_manager.h | 10 ++++------ include/kv_options.h | 6 ++++-- 6 files changed, 40 insertions(+), 28 deletions(-) diff --git a/benchmark/interference_bench.cpp b/benchmark/interference_bench.cpp index 69d573d10..ff1476b35 100644 --- a/benchmark/interference_bench.cpp +++ b/benchmark/interference_bench.cpp @@ -403,18 +403,18 @@ void ReportQosDelta(const char *name, const eloqstore::IoQosStats &begin, const eloqstore::IoQosStats &end, size_t shard, + uint32_t data_page_size, double secs) { auto d = [](uint64_t b, uint64_t e) { return e - b; }; - // Store-issued page IO in MB/s over the phase: every page metered by the - // budgets (user data + compaction relocations + index pages). This is - // the device-facing rate to compare against the fio calibration curve — - // unlike the workload's logical write MB/s, it includes background - // amplification (and excludes only manifest/fdatasync/segment IO). + // Budgeted page IO in MB/s over the phase: user data, compaction + // relocations, and index pages. This is not total device traffic; metadata, + // manifest, bulk file/snapshot, fdatasync, and segment IO are unbudgeted. auto mbps = [&](const eloqstore::IoQosStats::Budget &b, const eloqstore::IoQosStats::Budget &e) { - return secs > 0 ? (d(b.admitted_pages_, e.admitted_pages_) * 4096) / + return secs > 0 ? (d(b.admitted_pages_, e.admitted_pages_) * + static_cast(data_page_size)) / (secs * (1 << 20)) : 0.0; }; @@ -423,17 +423,18 @@ void ReportQosDelta(const char *name, << d(begin.read_.blocked_count_, end.read_.blocked_count_) << " read_blocked_us=" << d(begin.read_.blocked_us_, end.read_.blocked_us_) - << " read_page_mbps=" << mbps(begin.read_, end.read_) + << " read_budgeted_page_mbps=" << mbps(begin.read_, end.read_) << " bg_read_hwm=" << end.bg_read_.high_watermark_ << " bg_read_blocked=" << d(begin.bg_read_.blocked_count_, end.bg_read_.blocked_count_) << " bg_read_blocked_us=" << d(begin.bg_read_.blocked_us_, end.bg_read_.blocked_us_) - << " bg_read_page_mbps=" << mbps(begin.bg_read_, end.bg_read_) + << " bg_read_budgeted_page_mbps=" + << mbps(begin.bg_read_, end.bg_read_) << " write_hwm=" << end.write_.high_watermark_ << " write_blocked=" << d(begin.write_.blocked_count_, end.write_.blocked_count_) - << " write_page_mbps=" << mbps(begin.write_, end.write_) + << " write_budgeted_page_mbps=" << mbps(begin.write_, end.write_) << " fdatasync=" << d(begin.fdatasync_count_, end.fdatasync_count_) << " fdatasync_us=" << d(begin.fdatasync_us_, end.fdatasync_us_); @@ -522,9 +523,18 @@ int main(int argc, char *argv[]) << static_cast(achieved_write_pct + 0.5); for (size_t s = 0; s < num_shards; s++) { - ReportQosDelta( - "baseline", qos_start[s], qos_mid[s], s, FLAGS_baseline_secs); - ReportQosDelta("mixed", qos_mid[s], qos_end[s], s, FLAGS_storm_secs); + ReportQosDelta("baseline", + qos_start[s], + qos_mid[s], + s, + options.data_page_size, + FLAGS_baseline_secs); + ReportQosDelta("mixed", + qos_mid[s], + qos_end[s], + s, + options.data_page_size, + FLAGS_storm_secs); } store.Stop(); diff --git a/docs/architecture/07-io-stack.md b/docs/architecture/07-io-stack.md index 9dcd88a24..3b3b19660 100644 --- a/docs/architecture/07-io-stack.md +++ b/docs/architecture/07-io-stack.md @@ -48,9 +48,10 @@ Responsibilities: distinguishes budgeted page reads from metadata ops via the `KvTaskPageRead`/`BaseReqPageRead` user-data types. A cap of 0 disables a budget; a single request costlier than the cap is admitted alone once the - budget drains. Metadata, manifest, and segment IO are exempt. With - `enable_data_page_cache`, `max_inflight_write` also bounds cached-page pins - retained by write promotion until the corresponding IO completes. + budget drains. Metadata, manifest, bulk file/snapshot paths (`ReadFile`, + `ReadFilePrefix`, `WriteSnapshot`), `Fdatasync`, and segment IO are exempt. + With `enable_data_page_cache`, `max_inflight_write` also bounds cached-page + pins retained by write promotion until the corresponding IO completes. The read budget carries a **background sub-budget** (`bg_read_ratio` percent of `max_inflight_read`): page reads from tasks where `KvTask::IsBackground()` (BatchWrite, BackgroundWrite, EvictFile, Prewarm) diff --git a/docs/design/io_qos.md b/docs/design/io_qos.md index 9e4ae0108..6bfe40722 100644 --- a/docs/design/io_qos.md +++ b/docs/design/io_qos.md @@ -136,8 +136,9 @@ separately. Segment IO (`ReadSegments` / `WriteSegments`, zero-copy large values) is **out of scope** — see Non-Goals. -Metadata operations (open, statx, rename, unlink, mkdir) are exempt; their -burst size is already bounded by the 128-op chunking from #455. +Metadata operations (open, statx, rename, unlink, mkdir), manifest IO, and bulk +file/snapshot paths (`ReadFile`, `ReadFilePrefix`, `WriteSnapshot`) are exempt; +metadata burst size is already bounded by the 128-op chunking from #455. `fdatasync` is not counted initially but is instrumented (see Evaluation). Enforcement follows the existing idiom, per class (read or write). The @@ -189,8 +190,8 @@ class ever appears, split the write budget then.) Read budget structure: - Foreground reads: `inflight_read_pages_ ≤ max_inflight_read`. -- Background reads (compaction move batches, batch-write tree-traversal - reads, GC/upload-path reads): additionally +- Background page reads (compaction move batches, batch-write tree-traversal + reads, GC/prewarm page reads): additionally `bg_inflight_read_pages_ ≤ bg_read_limit` (a fraction of `max_inflight_read`, e.g. 25%). diff --git a/docs/design/io_qos_impl_plan.md b/docs/design/io_qos_impl_plan.md index 74dc7cdce..df05805ff 100644 --- a/docs/design/io_qos_impl_plan.md +++ b/docs/design/io_qos_impl_plan.md @@ -99,7 +99,7 @@ never by the blocked task) and unreachable in practice with caps ≪ | `IouringMgr::SubmitMergedWrite` (~777) | `write_budget_.Acquire(bytes / data_page_size)` after `merged_write_req_pool_->Alloc`, before `GetSQE`. | Exempt (unchanged): all metadata ops, manifest IO, `ReadFile` / -`WriteSnapshot` bulk paths, `Fdatasync` (instrumented only), +`ReadFilePrefix` / `WriteSnapshot` bulk paths, `Fdatasync` (instrumented only), `ReadSegments` / `WriteSegments` (out of scope per design Non-Goals). ### Release point diff --git a/include/async_io_manager.h b/include/async_io_manager.h index 12844a834..a34b6ae8e 100644 --- a/include/async_io_manager.h +++ b/include/async_io_manager.h @@ -69,12 +69,10 @@ struct IoQosStats uint32_t high_watermark_{0}; // max pages ever admitted uint64_t blocked_count_{0}; // acquisitions that had to wait uint64_t blocked_us_{0}; // cumulative wait time - // Cumulative pages ever admitted. For the write budget this is the - // store-issued device write volume in pages (user data pages, - // compaction relocations, index pages — everything metered by the - // budget), i.e. the store-side counterpart of device write MB/s for - // comparing against the fio calibration curve. Excludes manifest - // appends, fdatasync, and segment IO (unbudgeted). + // Cumulative configured data pages ever admitted by this budget. This + // is budgeted page-IO volume, not total device traffic. Metadata, + // manifest, bulk file/snapshot, fdatasync, and segment IO are + // unbudgeted and therefore absent. uint64_t admitted_pages_{0}; }; Budget read_; diff --git a/include/kv_options.h b/include/kv_options.h index 698ddc23f..a5817d210 100644 --- a/include/kv_options.h +++ b/include/kv_options.h @@ -79,6 +79,8 @@ struct KvOptions * page, so the cap means the same thing in append and non-append mode. * Also sizes the write request pools. With `enable_data_page_cache`, it * bounds cached-page write-promotion pins held until IO completion. + * Metadata, manifest, bulk file/snapshot, fdatasync, and segment IO are + * exempt. * Cannot be zero. * * NOTE: before the IO QoS work this option only sized the non-append @@ -90,8 +92,8 @@ struct KvOptions /** * @brief Per-shard cap on in-flight page-read IO, in configured * data-page units (`data_page_size`; docs/design/io_qos.md M1). Applies to - * data-page reads - * (ReadPage/ReadPages); metadata and segment IO are exempt. + * data-page reads (ReadPage/ReadPages); metadata, manifest, bulk file + * reads (ReadFile/ReadFilePrefix), and segment IO are exempt. * 0 disables the read budget. * * This is the device-calibration knob of the QoS sizing contract (see From ff311a9d0727b7b787f33247b2d2123afed2b343 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 16 Jul 2026 08:37:33 +0000 Subject: [PATCH 10/30] docs(io): correct M2 page-read producers --- docs/architecture/07-io-stack.md | 21 +++++++++++---------- docs/design/io_qos.md | 12 +++++++----- include/kv_options.h | 12 +++++++----- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/docs/architecture/07-io-stack.md b/docs/architecture/07-io-stack.md index 3b3b19660..306cede05 100644 --- a/docs/architecture/07-io-stack.md +++ b/docs/architecture/07-io-stack.md @@ -53,16 +53,17 @@ Responsibilities: With `enable_data_page_cache`, `max_inflight_write` also bounds cached-page pins retained by write promotion until the corresponding IO completes. The read budget carries a **background sub-budget** (`bg_read_ratio` - percent of `max_inflight_read`): page reads from tasks where - `KvTask::IsBackground()` (BatchWrite, BackgroundWrite, EvictFile, Prewarm) - are additionally bounded by it, so compaction/GC/batch-write read bursts - cannot crowd foreground point reads out of the device queue. Foreground may - use the entire read budget while background has no pending demand. Once a - background acquisition enters the wait path, its unused sub-budget stays - reserved through admission, including the wake-to-admit gap. Each class - waits on its own FIFO zone; release wakes background first and always wakes - foreground. The write budget has no split — all page writes come from write - tasks, i.e. background. + percent of `max_inflight_read`): budgeted page reads from `BatchWrite` and + `BackgroundWrite` (compaction) tasks are additionally bounded by it, so they + cannot crowd foreground point reads out of the device queue. `EvictFile` and + `Prewarm` are background task types, but local-GC `ReadFile` and + prewarm/download whole-file bulk IO remain exempt. Foreground may use the + entire read budget while background has no pending demand. Once a background + acquisition enters the wait path, its unused sub-budget stays reserved + through admission, including the wake-to-admit gap. Each class waits on its + own FIFO zone; release wakes background first and always wakes foreground. + The write budget has no split — all page writes come from write tasks, i.e. + background. `GetIoQosStats()` (also surfaced as `EloqStore::GetIoQosStats(shard_id)`) exposes in-flight/high-watermark/blocked counters (total read, bg-read slice, write) plus write-path fdatasync count and latency. The blocked fields are diff --git a/docs/design/io_qos.md b/docs/design/io_qos.md index 6bfe40722..75aca99f4 100644 --- a/docs/design/io_qos.md +++ b/docs/design/io_qos.md @@ -179,7 +179,8 @@ task-type predicate. Add `KvTask::IsBackground()`: - Foreground: `Read`, `Scan`, `ListObject`, `ListStandbyPartition`, `Reopen`. Do **not** reuse `ReadOnly()` — `EvictFile` and `Prewarm` are read-only but -background. +background. They do not currently issue budgeted page reads: local GC uses +`ReadFile`, while prewarm/download uses whole-file bulk IO; both remain exempt. The FG/BG split only applies to the **read** budget. All page writes are issued by write tasks (`BatchWrite`, `BackgroundWrite`), which are background @@ -190,8 +191,8 @@ class ever appears, split the write budget then.) Read budget structure: - Foreground reads: `inflight_read_pages_ ≤ max_inflight_read`. -- Background page reads (compaction move batches, batch-write tree-traversal - reads, GC/prewarm page reads): additionally +- Background page reads (compaction move batches and batch-write tree-traversal + reads): additionally `bg_inflight_read_pages_ ≤ bg_read_limit` (a fraction of `max_inflight_read`, e.g. 25%). @@ -355,8 +356,9 @@ All budgets are per shard, but the device is shared by all shards waiters — no cross-thread coroutine resumption needed. - Cloud mode: NIC bandwidth is shared between background uploads and foreground cache-miss downloads; the same FG/BG discipline should - eventually apply to cloud slots. Upload-path bulk disk reads - (`ReadFilePrefix`) remain exempt from the page-IO budget. + eventually apply to cloud slots. Whole-file upload, prewarm/download, and + snapshot disk IO remains exempt from the page-IO budget, including + `ReadFile`, `ReadFilePrefix`, and `WriteSnapshot` bulk paths. ## Non-Goals diff --git a/include/kv_options.h b/include/kv_options.h index a5817d210..4ca19c71b 100644 --- a/include/kv_options.h +++ b/include/kv_options.h @@ -92,8 +92,9 @@ struct KvOptions /** * @brief Per-shard cap on in-flight page-read IO, in configured * data-page units (`data_page_size`; docs/design/io_qos.md M1). Applies to - * data-page reads (ReadPage/ReadPages); metadata, manifest, bulk file - * reads (ReadFile/ReadFilePrefix), and segment IO are exempt. + * data-page reads (ReadPage/ReadPages). Local-GC ReadFile and + * prewarm/download whole-file bulk IO remain exempt, as do metadata, + * manifest, and segment IO. * 0 disables the read budget. * * This is the device-calibration knob of the QoS sizing contract (see @@ -106,9 +107,10 @@ struct KvOptions uint32_t max_inflight_read = 64; /** * @brief Background share of max_inflight_read, in percent (clamped to - * 1..100; docs/design/io_qos.md M2). Page reads issued by background - * tasks (batch write, compaction, GC, prewarm) are bounded by this - * sub-budget so they cannot crowd out foreground point reads. + * 1..100; docs/design/io_qos.md M2). Page reads issued by batch-write and + * compaction tasks are bounded by this sub-budget so they cannot crowd out + * foreground point reads. Local GC and prewarm/download use exempt + * whole-file bulk IO instead. * Foreground reads may use the entire read budget while no background * acquisition is pending; pending background demand reserves its unused * share through admission. No effect when the read budget is disabled From 7d7194827cf0d6579f1d351f10af46f05c0706bd Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 16 Jul 2026 09:25:45 +0000 Subject: [PATCH 11/30] Fix interference accounting and periodic QoS gauges --- benchmark/interference_bench.cpp | 20 ++++++---- docs/architecture/07-io-stack.md | 2 +- docs/design/io_qos_impl_plan.md | 2 +- include/eloqstore_metrics.h | 5 +-- include/storage/shard.h | 8 ++-- src/storage/shard.cpp | 64 +++++++++++++++----------------- 6 files changed, 51 insertions(+), 50 deletions(-) diff --git a/benchmark/interference_bench.cpp b/benchmark/interference_bench.cpp index ff1476b35..16619fd66 100644 --- a/benchmark/interference_bench.cpp +++ b/benchmark/interference_bench.cpp @@ -199,7 +199,8 @@ void ReadLoop(eloqstore::EloqStore *store, std::string_view(reader->key_, sizeof(key))); reader->issue_phase_ = g_phase.load(std::memory_order_relaxed); reader->start_us_ = utils::UnixTs(); - store->ExecAsyn(&reader->request_, uint64_t(reader), callback); + CHECK(store->ExecAsyn(&reader->request_, uint64_t(reader), callback)) + << "read issue rejected; fixed-depth result is invalid"; }; size_t inflight = 0; @@ -215,13 +216,16 @@ void ReadLoop(eloqstore::EloqStore *store, : nullptr; if (dst != nullptr) { - dst->samples.push_back(lat); - if (reader->request_.Error() == eloqstore::KvError::NotFound) + if (reader->request_.Error() == eloqstore::KvError::NoError) + { + dst->samples.push_back(lat); + } + else if (reader->request_.Error() == + eloqstore::KvError::NotFound) { dst->not_found++; } - else if (reader->request_.Error() != - eloqstore::KvError::NoError) + else { dst->errors++; } @@ -337,7 +341,8 @@ StormTotals StormLoop(eloqstore::EloqStore *store) for (auto &w : writers) { NextStormBatch(*w); - store->ExecAsyn(&w->request_, uint64_t(w.get()), callback); + CHECK(store->ExecAsyn(&w->request_, uint64_t(w.get()), callback)) + << "initial storm issue rejected; fixed-depth result is invalid"; } size_t inflight = writers.size(); while (inflight > 0) @@ -354,7 +359,8 @@ StormTotals StormLoop(eloqstore::EloqStore *store) continue; } NextStormBatch(*w); - store->ExecAsyn(&w->request_, uint64_t(w), callback); + CHECK(store->ExecAsyn(&w->request_, uint64_t(w), callback)) + << "storm reissue rejected; fixed-depth result is invalid"; } StormTotals total; for (auto &w : writers) diff --git a/docs/architecture/07-io-stack.md b/docs/architecture/07-io-stack.md index 306cede05..84165c45a 100644 --- a/docs/architecture/07-io-stack.md +++ b/docs/architecture/07-io-stack.md @@ -43,7 +43,7 @@ Responsibilities: `read_budget_` (`max_inflight_read`; `ReadPage`/`ReadPages`, per-page acquisition so a batch larger than the cap cannot deadlock) and `write_budget_` (`max_inflight_write`; `WritePage` cost 1, - `SubmitMergedWrite` cost `bytes / data_page_size`). Budget is acquired + `SubmitMergedWrite` cost `ceil(bytes / data_page_size)`). Budget is acquired immediately before SQE prep and released per CQE in `PollComplete`, which distinguishes budgeted page reads from metadata ops via the `KvTaskPageRead`/`BaseReqPageRead` user-data types. A cap of 0 disables a diff --git a/docs/design/io_qos_impl_plan.md b/docs/design/io_qos_impl_plan.md index df05805ff..1beb5655d 100644 --- a/docs/design/io_qos_impl_plan.md +++ b/docs/design/io_qos_impl_plan.md @@ -96,7 +96,7 @@ never by the blocked task) and unreachable in practice with caps ≪ | `IouringMgr::ReadPage` (`async_io_manager.cpp` ~484) | `read_budget_.Acquire(1)` at the top of the `read_page` lambda's retry loop, before `GetSQE`. Release is per-CQE, so each retry re-acquires. | | `IouringMgr::ReadPages` (~565) | Acquire **per page** inside `send_req`, not per batch. Rationale: a 128-page compaction batch must not deadlock against a BG sub-budget of 64 (commit 2); per-page acquisition lets the task block mid-batch while already-submitted pages complete. This supersedes the "atomic batch acquire" idea discussed during design review. | | `IouringMgr::WritePage` (~742) | `write_budget_.Acquire(1)` after `write_req_pool_->Alloc`, before `GetSQE`. | -| `IouringMgr::SubmitMergedWrite` (~777) | `write_budget_.Acquire(bytes / data_page_size)` after `merged_write_req_pool_->Alloc`, before `GetSQE`. | +| `IouringMgr::SubmitMergedWrite` (~777) | `write_budget_.Acquire(ceil(bytes / data_page_size))` after `merged_write_req_pool_->Alloc`, before `GetSQE`. | Exempt (unchanged): all metadata ops, manifest IO, `ReadFile` / `ReadFilePrefix` / `WriteSnapshot` bulk paths, `Fdatasync` (instrumented only), diff --git a/include/eloqstore_metrics.h b/include/eloqstore_metrics.h index f39e96c3f..c1c99dda2 100644 --- a/include/eloqstore_metrics.h +++ b/include/eloqstore_metrics.h @@ -54,9 +54,8 @@ inline const Name NAME_ELOQSTORE_INFLIGHT_BG_READ_PAGES{ inline const Name NAME_ELOQSTORE_INFLIGHT_WRITE_PAGES{ "eloqstore_inflight_write_pages"}; -// Collection interval for Phase 9-11 gauge metrics (index buffer pool, page -// pool, open file, local space) These metrics are collected every N -// WorkOneRound() calls to reduce overhead +// Periodic scheduler gauges are collected every N active shard rounds to +// reduce overhead. inline constexpr size_t ELOQSTORE_GAUGE_COLLECTION_INTERVAL = 1000; } // namespace metrics #endif // ELOQSTORE_WITH_TXSERVICE diff --git a/include/storage/shard.h b/include/storage/shard.h index 9fb9cf28a..5df5698c7 100644 --- a/include/storage/shard.h +++ b/include/storage/shard.h @@ -104,6 +104,9 @@ class Shard private: void WorkLoop(); void InitIoMgrAndPagePool(); +#ifdef ELOQSTORE_WITH_TXSERVICE + void CollectPeriodicGauges(metrics::Meter *meter); +#endif bool ExecuteReadyTasks(); void OnTaskFinished(KvTask *task); void RetryOomRequest(KvRequest *req); @@ -285,9 +288,8 @@ class Shard #endif #ifdef ELOQSTORE_WITH_TXSERVICE - size_t work_one_round_count_{ - 0}; // Counter for frequency-controlled metric collection (not atomic - // since each Shard runs in single-threaded context) + // Not atomic: both scheduler modes execute one shard serially. + size_t gauge_collection_round_count_{0}; #endif // TSC frequency in cycles per microsecond (measured at initialization) diff --git a/src/storage/shard.cpp b/src/storage/shard.cpp index ec578640a..b78a2d6ec 100644 --- a/src/storage/shard.cpp +++ b/src/storage/shard.cpp @@ -122,6 +122,33 @@ void Shard::InitIoMgrAndPagePool() io_mgr_and_page_pool_inited_.store(true, std::memory_order_release); } +#ifdef ELOQSTORE_WITH_TXSERVICE +void Shard::CollectPeriodicGauges(metrics::Meter *meter) +{ + if (++gauge_collection_round_count_ % + metrics::ELOQSTORE_GAUGE_COLLECTION_INTERVAL != + 0) + { + return; + } + + meter->Collect(metrics::NAME_ELOQSTORE_INDEX_BUFFER_POOL_USED, + static_cast(index_mgr_.GetBufferPoolUsed())); + meter->Collect(metrics::NAME_ELOQSTORE_OPEN_FILE_COUNT, + static_cast(io_mgr_->GetOpenFileCount())); + meter->Collect(metrics::NAME_ELOQSTORE_LOCAL_SPACE_USED, + static_cast(io_mgr_->GetLocalSpaceUsed())); + + const IoQosStats qos = io_mgr_->GetIoQosStats(); + meter->Collect(metrics::NAME_ELOQSTORE_INFLIGHT_READ_PAGES, + static_cast(qos.read_.inflight_)); + meter->Collect(metrics::NAME_ELOQSTORE_INFLIGHT_BG_READ_PAGES, + static_cast(qos.bg_read_.inflight_)); + meter->Collect(metrics::NAME_ELOQSTORE_INFLIGHT_WRITE_PAGES, + static_cast(qos.write_.inflight_)); +} +#endif + void Shard::WorkLoop() { shard = this; @@ -257,6 +284,7 @@ void Shard::WorkLoop() metrics::NAME_ELOQSTORE_WORK_ONE_ROUND_DURATION, round_start); meter->Collect(metrics::NAME_ELOQSTORE_TASK_MANAGER_ACTIVE_TASKS, static_cast(task_mgr_.NumActive())); + CollectPeriodicGauges(meter); } #endif } @@ -1388,41 +1416,7 @@ void Shard::WorkOneRound() round_start); meter->Collect(metrics::NAME_ELOQSTORE_TASK_MANAGER_ACTIVE_TASKS, static_cast(task_mgr_.NumActive())); - - // Increment round counter for frequency-controlled metric collection - work_one_round_count_++; - bool should_collect_gauges = - (work_one_round_count_ % - metrics::ELOQSTORE_GAUGE_COLLECTION_INTERVAL) == 0; - - // Collect used/count metrics (frequency-controlled) - // Note: limit metrics are collected once at initialization in Start() - if (should_collect_gauges) - { - // Collect index buffer pool used - size_t index_used = index_mgr_.GetBufferPoolUsed(); - meter->Collect(metrics::NAME_ELOQSTORE_INDEX_BUFFER_POOL_USED, - static_cast(index_used)); - - // Collect open file count - size_t open_file_count = io_mgr_->GetOpenFileCount(); - meter->Collect(metrics::NAME_ELOQSTORE_OPEN_FILE_COUNT, - static_cast(open_file_count)); - - // Collect local space used - size_t local_space_used = io_mgr_->GetLocalSpaceUsed(); - meter->Collect(metrics::NAME_ELOQSTORE_LOCAL_SPACE_USED, - static_cast(local_space_used)); - - // Collect in-flight page-IO budget usage (io_qos.md M1) - IoQosStats qos = io_mgr_->GetIoQosStats(); - meter->Collect(metrics::NAME_ELOQSTORE_INFLIGHT_READ_PAGES, - static_cast(qos.read_.inflight_)); - meter->Collect(metrics::NAME_ELOQSTORE_INFLIGHT_BG_READ_PAGES, - static_cast(qos.bg_read_.inflight_)); - meter->Collect(metrics::NAME_ELOQSTORE_INFLIGHT_WRITE_PAGES, - static_cast(qos.write_.inflight_)); - } + CollectPeriodicGauges(meter); } #endif } From 3c2fb9b0375bd79f824be444ee60b0d7c2035958 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 16 Jul 2026 12:07:54 +0000 Subject: [PATCH 12/30] fix(io): close remaining QoS review gaps Preserve a fully reserved background read slice across the wake-to-admit gap and cover all page-read CQE error paths. Expose stable QoS stats, harden option parsing and benchmark validity, and add deterministic accounting and scheduler regressions. --- benchmark/eloq_store_bm.cc | 13 +- benchmark/eloq_store_bm.h | 7 +- benchmark/interference_bench.cpp | 102 ++++++++++++---- docs/design/io_qos.md | 11 +- include/async_io_manager.h | 31 ----- include/eloq_store.h | 7 +- include/fail_point.h | 47 ++++++- include/types.h | 31 +++++ scripts/io_calibration_sweep.sh | 6 +- src/async_io_manager.cpp | 48 ++++++-- src/kv_options.cpp | 4 +- tests/common.cpp | 17 ++- tests/common.h | 8 +- tests/eloq_store_test.cpp | 41 +++++++ tests/io_qos.cpp | 203 ++++++++++++++++++++++++++----- 15 files changed, 436 insertions(+), 140 deletions(-) diff --git a/benchmark/eloq_store_bm.cc b/benchmark/eloq_store_bm.cc index 0657d810a..53878401c 100644 --- a/benchmark/eloq_store_bm.cc +++ b/benchmark/eloq_store_bm.cc @@ -550,7 +550,6 @@ struct Get2Client moodycamel::BlockingConcurrentQueue done_; std::vector lat_us_; uint64_t outstanding_{0}; - uint64_t successes_{0}; uint64_t read_failed_{0}; uint64_t issue_failed_{0}; }; @@ -566,7 +565,8 @@ uint64_t Get2NowUs() void Benchmark::OnReadV2(::eloqstore::KvRequest *req) { auto *op = reinterpret_cast(req->UserData()); - static_cast(op->client_)->done_.enqueue(op); + CHECK(static_cast(op->client_)->done_.enqueue(op)) + << "GET2 completion queue allocation failed"; } void Benchmark::RunGet2(uint32_t client_threads, @@ -628,7 +628,7 @@ void Benchmark::RunGet2(uint32_t client_threads, } std::vector shard_out(nshards, 0); - auto issue = [&](ReadOperation *op) -> bool + auto issue = [&](ReadOperation *op) { uint64_t key_index = gen.get_key_index(OBJECT_GENERATOR_KEY_RANDOM); @@ -665,11 +665,10 @@ void Benchmark::RunGet2(uint32_t client_threads, OnReadV2)) { ++me.issue_failed_; - return false; + return; } ++shard_out[op->shard_]; ++me.outstanding_; - return true; }; auto complete = [&](ReadOperation *op) @@ -684,7 +683,6 @@ void Benchmark::RunGet2(uint32_t client_threads, return; } me.lat_us_.push_back(Get2NowUs() - op->start_ts_); - ++me.successes_; }; for (auto &op : ops) @@ -727,16 +725,15 @@ void Benchmark::RunGet2(uint32_t client_threads, const double dur_sec = (Get2NowUs() - bench_start) / 1e6; std::vector all; - uint64_t successes = 0; uint64_t read_failures = 0; uint64_t issue_failures = 0; for (auto &cl : clients) { - successes += cl.successes_; read_failures += cl.read_failed_; issue_failures += cl.issue_failed_; all.insert(all.end(), cl.lat_us_.begin(), cl.lat_us_.end()); } + const uint64_t successes = all.size(); std::sort(all.begin(), all.end()); auto pct = [&](double p) -> uint64_t { diff --git a/benchmark/eloq_store_bm.h b/benchmark/eloq_store_bm.h index 9f35273d4..0a2a904d6 100644 --- a/benchmark/eloq_store_bm.h +++ b/benchmark/eloq_store_bm.h @@ -133,12 +133,7 @@ struct ReadOperation explicit ReadOperation(const Benchmark *bm); ReadOperation(const ReadOperation &rhs) = delete; - ReadOperation(ReadOperation &&rhs) - : req_(std::move(rhs.req_)), - key_(std::move(rhs.key_)), - start_ts_(rhs.start_ts_) - { - } + ReadOperation(ReadOperation &&rhs) noexcept = default; req_uptr req_; std::string key_; diff --git a/benchmark/interference_bench.cpp b/benchmark/interference_bench.cpp index 16619fd66..be090d101 100644 --- a/benchmark/interference_bench.cpp +++ b/benchmark/interference_bench.cpp @@ -12,10 +12,12 @@ * storm control per-file liveness exactly. * 2. baseline — closed-loop uniform-random point reads at fixed * concurrency for baseline_secs. No writes. - * 3. mixed — the measured workload: a write-dominated op mix (default + * 3. mixed — the measured workload: a write-dominated op mix (target * 90% write key-ops / 10% point reads, --write_read_ratio) - * where reads are paced off completed write batches so the - * ratio holds regardless of relative speeds; set + * where completed write batches grant read credits. The + * ratio is a read upper bound: slow readers can make the + * achieved mix more write-heavy, which achieved_write_pct + * reports. Set * write_read_ratio=0 for the original unthrottled * reads-vs-storm shape. The writes overwrite a rotating * strided subset of keys (span of every ratio, default 3 @@ -30,7 +32,9 @@ * (computed from raw samples, not a sliding window), the storm's write MB/s, * and per-shard IoQosStats deltas (in-flight watermarks, budget-blocked * counts/time, fdatasync). Greppable one-line summaries are prefixed with - * "RESULT" for sweep scripts. + * "RESULT" for sweep scripts. A run exits nonzero if either measured phase + * has errors, missing keys, fewer than --min_read_samples successes, or an + * enabled BG budget records no mixed-phase background page reads. * * EloqStore options (including the QoS knobs) come from --kvoptions ini, so * sweeps only vary the ini / flags. See opts_interference.ini. @@ -47,7 +51,6 @@ #include #include -#include "async_io_manager.h" // IoQosStats #include "coding.h" #include "eloq_store.h" #include "utils.h" @@ -70,10 +73,13 @@ DEFINE_uint32(storm_span, 3, "keys overwritten within each stride"); DEFINE_uint32(storm_batch_keys, 2048, "keys per storm batch-write request"); DEFINE_uint32(write_read_ratio, 9, - "write key-ops per point read in the mixed phase (9 = 90/10 " - "write/read op mix). 0 = reads run unthrottled closed-loop " - "alongside the writes (the original storm shape)"); + "completed write key-ops required to grant one mixed-phase " + "read (9 targets at least 90% writes; slow readers can make it " + "more write-heavy). 0 = unthrottled closed-loop reads"); DEFINE_bool(load, true, "load data first (false reuses an existing store)"); +DEFINE_uint64(min_read_samples, + 1000, + "minimum successful reads required in each measured phase"); using namespace std::chrono; @@ -84,9 +90,9 @@ constexpr char kTable[] = "ifb"; std::atomic g_phase{0}; // 0 load, 1 baseline, 2 mixed, 3 done // Read pacing for the mixed phase: every completed storm batch adds its key -// count; issuing one read consumes write_read_ratio credits, so reads track -// the write throughput at the configured op ratio (e.g. 9 -> 10% reads / -// 90% writes by ops). +// count; issuing one read consumes write_read_ratio credits, allowing at most +// one read per ratio completed writes. Slow readers can make the actual mix +// more write-heavy; achieved_write_pct reports it. std::atomic g_read_credits{0}; bool TryConsumeReadCredits() @@ -121,10 +127,6 @@ std::string MakeKey(uint64_t key) struct Reader { - explicit Reader(uint32_t id) : id_(id) - { - } - const uint32_t id_; eloqstore::ReadRequest request_; char key_[sizeof(uint64_t)]; uint64_t start_us_{0}; @@ -153,24 +155,43 @@ void ReportPhase(const char *name, PhaseLatencies &lat, double secs) std::sort(lat.samples.begin(), lat.samples.end()); const size_t n = lat.samples.size(); const uint64_t qps = secs > 0 ? static_cast(n / secs) : 0; + const uint64_t p99 = Percentile(lat.samples, 0.99); LOG(INFO) << "RESULT phase=" << name << " reads=" << n << " qps=" << qps << " p50=" << Percentile(lat.samples, 0.50) - << " p90=" << Percentile(lat.samples, 0.90) - << " p99=" << Percentile(lat.samples, 0.99) + << " p90=" << Percentile(lat.samples, 0.90) << " p99=" << p99 << " p999=" << Percentile(lat.samples, 0.999) << " max=" << (n ? lat.samples.back() : 0) << " not_found=" << lat.not_found << " errors=" << lat.errors << " (latency us)"; } +bool ValidatePhase(const char *name, const PhaseLatencies &lat) +{ + bool valid = true; + if (lat.samples.size() < FLAGS_min_read_samples) + { + LOG(ERROR) << name << " phase produced only " << lat.samples.size() + << " successful reads; require at least " + << FLAGS_min_read_samples; + valid = false; + } + if (lat.not_found != 0 || lat.errors != 0) + { + LOG(ERROR) << name << " phase had not_found=" << lat.not_found + << " errors=" << lat.errors; + valid = false; + } + return valid; +} + /** * Read driver. Baseline phase: closed loop at FLAGS_read_concurrency. * Mixed phase with write_read_ratio > 0: reads are additionally gated on - * credits produced by completed storm writes, holding the op mix at - * 1 read : ratio writes (reads pause when writes stall, and vice versa - * never outrun the ratio). Each completion is recorded into the phase the - * request was ISSUED in (so a request straddling a phase flip does not - * contaminate the other phase). + * credits produced by completed storm writes, preventing reads from exceeding + * 1 read per ratio writes. Writers are not paced by readers, so slow readers + * can produce a more write-heavy mix. Each completion is recorded into the + * phase the request was ISSUED in (so a request straddling a phase flip does + * not contaminate the other phase). */ void ReadLoop(eloqstore::EloqStore *store, PhaseLatencies *baseline, @@ -182,12 +203,15 @@ void ReadLoop(eloqstore::EloqStore *store, idle.reserve(FLAGS_read_concurrency); for (uint32_t i = 0; i < FLAGS_read_concurrency; i++) { - readers[i] = std::make_unique(i); + readers[i] = std::make_unique(); idle.push_back(readers[i].get()); } auto callback = [&finished](eloqstore::KvRequest *req) - { finished.enqueue(reinterpret_cast(req->UserData())); }; + { + CHECK(finished.enqueue(reinterpret_cast(req->UserData()))) + << "read completion queue allocation failed"; + }; std::mt19937_64 rnd(12345); auto send_req = [&](Reader *reader) @@ -271,7 +295,6 @@ struct StormWriter uint64_t bytes_written_{0}; uint64_t keys_written_{0}; uint32_t batch_keys_{0}; // keys in the currently in-flight batch - bool done_{false}; }; /** @@ -336,7 +359,11 @@ StormTotals StormLoop(eloqstore::EloqStore *store) writers[i] = std::make_unique(i); } auto callback = [&finished](eloqstore::KvRequest *req) - { finished.enqueue(reinterpret_cast(req->UserData())); }; + { + CHECK( + finished.enqueue(reinterpret_cast(req->UserData()))) + << "storm completion queue allocation failed"; + }; for (auto &w : writers) { @@ -452,7 +479,12 @@ int main(int argc, char *argv[]) { google::ParseCommandLineFlags(&argc, &argv, true); CHECK_GT(FLAGS_partitions, 0u); + CHECK_GT(FLAGS_keys_per_partition, 0u); CHECK_GT(FLAGS_read_concurrency, 0u); + CHECK_GT(FLAGS_baseline_secs, 0u); + CHECK_GT(FLAGS_storm_secs, 0u); + CHECK_GT(FLAGS_storm_batch_keys, 0u); + CHECK_GT(FLAGS_min_read_samples, 0u); CHECK_GT(FLAGS_storm_span, 0u); CHECK_GT(FLAGS_storm_ratio, FLAGS_storm_span) << "storm must be a PARTIAL overwrite (span < ratio); a full " @@ -487,6 +519,7 @@ int main(int argc, char *argv[]) PhaseLatencies baseline, storm_lat; // Baseline phase: reads only. + uint64_t mixed_bg_read_pages = 0; for (size_t s = 0; s < num_shards; s++) { qos_start[s] = store.GetIoQosStats(s); @@ -541,8 +574,23 @@ int main(int argc, char *argv[]) s, options.data_page_size, FLAGS_storm_secs); + mixed_bg_read_pages += qos_end[s].bg_read_.admitted_pages_ - + qos_mid[s].bg_read_.admitted_pages_; + } + + const bool baseline_valid = ValidatePhase("baseline", baseline); + const bool mixed_valid = ValidatePhase("mixed", storm_lat); + bool interference_valid = true; + if (options.max_inflight_read != 0 && mixed_bg_read_pages == 0) + { + LOG(ERROR) << "mixed phase produced no budgeted background page " + "reads; the intended write/compaction interference was " + "not exercised"; + interference_valid = false; } + const bool valid = baseline_valid && mixed_valid && interference_valid; + LOG(INFO) << "RESULT validation=" << (valid ? "pass" : "fail"); store.Stop(); - return 0; + return valid ? 0 : 2; } diff --git a/docs/design/io_qos.md b/docs/design/io_qos.md index 75aca99f4..376229b5a 100644 --- a/docs/design/io_qos.md +++ b/docs/design/io_qos.md @@ -318,9 +318,12 @@ The two read-side options deliberately live at different levels: pools dwarf the ≤ max_inflight_write pages of pins; only tests noticed). - **`max_inflight_write` is redefined, not retired.** It is the M1 write cap (configured data-page units, default dropped from 32768 to 512). - `WriteReqPool` stays sized to it; in-flight writes can never exceed it, so - the pool bound and the QoS bound coincide. Note this is a behavioral - change for deployments that set the old option explicitly. + In-flight pages are bounded by `max(cap, one request's cost)` because one + oversized request may run alone to guarantee progress. `WriteReqPool` stays + numerically sized to the option, but counts request objects rather than page + units; it is therefore a conservative allocation bound, not the QoS bound + itself. Note this is a behavioral change for deployments that set the old + option explicitly. - **`max_write_concurrency` is no longer an IO-QoS knob.** Note it is enforced **per shard** (each shard's `TaskManager` counts its own active write tasks against the store-wide option value), so the device-wide task @@ -387,7 +390,7 @@ Export per-shard counters from day one; tuning must be measurement-driven: - Cumulative blocked time and counts per admission class: `read_` is foreground waits only, `bg_read_` is background waits, and `write_` is all write waits. - Background write bytes/sec (actual, vs. limit when M3 lands). -- fdatasync count and latency histogram. +- fdatasync count and cumulative batch wall time. ## Evaluation Plan diff --git a/include/async_io_manager.h b/include/async_io_manager.h index a34b6ae8e..bd0c194d5 100644 --- a/include/async_io_manager.h +++ b/include/async_io_manager.h @@ -54,37 +54,6 @@ class ManifestFile virtual KvError SkipPadding(size_t n) = 0; }; -/** - * @brief Per-shard IO QoS statistics (see docs/design/io_qos.md). - * - * Counters are single-writer relaxed atomics so tests/diagnostics can sample - * a race-free, non-coherent snapshot while the shard is live. Values are exact - * once the shard is quiesced. - */ -struct IoQosStats -{ - struct Budget - { - uint32_t inflight_{0}; // pages currently admitted - uint32_t high_watermark_{0}; // max pages ever admitted - uint64_t blocked_count_{0}; // acquisitions that had to wait - uint64_t blocked_us_{0}; // cumulative wait time - // Cumulative configured data pages ever admitted by this budget. This - // is budgeted page-IO volume, not total device traffic. Metadata, - // manifest, bulk file/snapshot, fdatasync, and segment IO are - // unbudgeted and therefore absent. - uint64_t admitted_pages_{0}; - }; - Budget read_; - // Background slice of read_ (M2), bounded by the bg sub-budget. Inflight, - // high-watermark, and admitted pages are subsets of read_; blocked fields - // are per-class (read_ is foreground, bg_read_ is background). - Budget bg_read_; - Budget write_; - uint64_t fdatasync_count_{0}; // write-path fdatasync ops (FdatasyncFiles) - uint64_t fdatasync_us_{0}; // cumulative batch wall time -}; - /** * @brief Per-shard in-flight page-IO budget (M1/M2 in docs/design/io_qos.md). * diff --git a/include/eloq_store.h b/include/eloq_store.h index 355f41e6e..a83c412e2 100644 --- a/include/eloq_store.h +++ b/include/eloq_store.h @@ -34,7 +34,6 @@ namespace eloqstore { class Shard; class EloqStore; -struct IoQosStats; enum class RequestType : uint8_t { @@ -986,9 +985,9 @@ class EloqStore /** * @brief Per-shard IO QoS statistics (in-flight page-IO budgets, - * fdatasync accounting; see docs/design/io_qos.md). Counters use - * single-writer relaxed atomics, yielding a race-free but non-coherent - * live snapshot and exact values once the shard is quiesced. + * fdatasync accounting; see docs/design/io_qos.md). Backing counters use + * single-writer relaxed atomics; the returned plain value is a race-free + * but non-coherent live snapshot and is exact once the shard is quiesced. * Returns zeros for invalid shard IDs or managers without budgets. */ IoQosStats GetIoQosStats(size_t shard_id) const; diff --git a/include/fail_point.h b/include/fail_point.h index cabe92efe..ebd5e7fe8 100644 --- a/include/fail_point.h +++ b/include/fail_point.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include // Test-only error injection. Unlike KillPoint (kill_point.h), which SIGTERMs @@ -55,20 +56,48 @@ class FailPoint // @p name must be a string literal (stored by pointer, not copied). void ArmOnce(const char *name) { - persistent_.store(false, std::memory_order_relaxed); - armed_.store(name, std::memory_order_release); + Arm(name, false, false); } // Arm until Disarm. Used by scheduler regressions that must perturb every // matching wake throughout a bounded observation window. void ArmPersistent(const char *name) { - persistent_.store(true, std::memory_order_relaxed); - armed_.store(name, std::memory_order_release); + Arm(name, true, false); + } + + // Arm a persistent point and hold a cooperatively-yielding action at its + // first wake until the test observes the exact scheduler state. The action + // calls MarkPauseReached() only after yielding once, then polls + // PauseRequested() between further yields so it never blocks the shard. + void ArmPersistentPaused(const char *name) + { + Arm(name, true, true); + } + + void ReleasePause() + { + paused_.store(false, std::memory_order_release); + } + + bool PauseRequested() const + { + return paused_.load(std::memory_order_acquire); + } + + void MarkPauseReached() + { + pause_reached_.store(true, std::memory_order_release); + } + + bool PauseReached() const + { + return pause_reached_.load(std::memory_order_acquire); } void Disarm() { + paused_.store(false, std::memory_order_release); armed_.store(nullptr, std::memory_order_release); } @@ -90,7 +119,17 @@ class FailPoint } private: + void Arm(const char *name, bool persistent, bool paused) + { + pause_reached_.store(false, std::memory_order_relaxed); + paused_.store(paused, std::memory_order_relaxed); + persistent_.store(persistent, std::memory_order_relaxed); + armed_.store(name, std::memory_order_release); + } + std::atomic armed_{nullptr}; std::atomic persistent_{false}; + std::atomic paused_{false}; + std::atomic pause_reached_{false}; }; } // namespace eloqstore diff --git a/include/types.h b/include/types.h index 01372c7b6..9e11baaa2 100644 --- a/include/types.h +++ b/include/types.h @@ -27,6 +27,37 @@ enum class StoreMode Cloud }; +/** + * @brief Plain snapshot of one shard's IO QoS statistics. + * + * The backing counters are single-writer relaxed atomics. Sampling them is + * race-free but not coherent across fields while the shard is live; values + * are exact once the shard is quiesced. See docs/design/io_qos.md. + */ +struct IoQosStats +{ + struct Budget + { + uint32_t inflight_{0}; // pages currently admitted + uint32_t high_watermark_{0}; // max pages ever admitted + uint64_t blocked_count_{0}; // acquisitions that had to wait + uint64_t blocked_us_{0}; // cumulative wait time + // Cumulative configured data pages ever admitted by this budget. This + // is budgeted page-IO volume, not total device traffic. Metadata, + // manifest, bulk file/snapshot, fdatasync, and segment IO are + // unbudgeted and therefore absent. + uint64_t admitted_pages_{0}; + }; + Budget read_; + // Background slice of read_ (M2), bounded by the bg sub-budget. Inflight, + // high-watermark, and admitted pages are subsets of read_; blocked fields + // are per-class (read_ is foreground, bg_read_ is background). + Budget bg_read_; + Budget write_; + uint64_t fdatasync_count_{0}; // write-path fdatasync ops (FdatasyncFiles) + uint64_t fdatasync_us_{0}; // cumulative batch wall time +}; + using PageId = uint32_t; constexpr PageId MaxPageId = UINT32_MAX; diff --git a/scripts/io_calibration_sweep.sh b/scripts/io_calibration_sweep.sh index 358d9fc23..5b90fb6b4 100755 --- a/scripts/io_calibration_sweep.sh +++ b/scripts/io_calibration_sweep.sh @@ -22,10 +22,8 @@ # total bytes written per step exceed the SLC cache if you want # steady-state numbers; watch for a step whose actual write MB/s sags # below the target — that is the fold-over signature. -# * Run against a FILE on the target filesystem (not the raw device) to -# include filesystem effects, or a raw block device for pure-device -# numbers. Never point this at a device with data you care about when -# using --device. +# * The script always runs against files below --dir, so the result includes +# filesystem effects. Use a dedicated directory on the target device. # # Requires: fio, python3. # diff --git a/src/async_io_manager.cpp b/src/async_io_manager.cpp index 8c9ad2889..5d6a03375 100644 --- a/src/async_io_manager.cpp +++ b/src/async_io_manager.cpp @@ -205,7 +205,15 @@ void IoBudget::Acquire(uint32_t cost, bool background) ? bg_cap_ - bg_inflight : 0; const uint32_t fg_cap = cap_ > reserved ? cap_ - reserved : 0; - return inflight + cost > fg_cap && inflight != 0; + const bool exceeds_fg_cap = + uint64_t{inflight} + cost > uint64_t{fg_cap}; + // Oversized requests may exceed the configured cap only when they are + // alone and no background entitlement is reserved. Without the + // reserved check, a foreground request can steal a 100% BG slice in + // the wake-to-admit gap whenever total inflight briefly reaches zero. + const bool oversized_alone = + inflight == 0 && reserved == 0 && cost > cap_; + return exceeds_fg_cap && !oversized_alone; }; // Each class queues behind its own existing waiters (approximate FIFO // across arrivals within the class). @@ -230,8 +238,25 @@ void IoBudget::Acquire(uint32_t cost, bool background) zone.Wait(ThdTask()); if (use_bg) { - TEST_FAIL_POINT_ACTION("IoBudgetBgWake", - ThdTask()->YieldToLowPQ()); + TEST_FAIL_POINT_ACTION("IoBudgetBgWake", { + FailPoint &fail_point = FailPoint::GetInstance(); + if (fail_point.PauseRequested() && + !fail_point.PauseReached()) + { + // Prove that the paused regression reached the exact + // wake gap with foreground demand both ready and + // queued, rather than relying on cumulative counters. + CHECK(!waiting_.Empty()); + CHECK_GT(shard->ready_tasks_.Size(), 0); + CHECK(shard->ready_tasks_.Peek()->Type() == + TaskType::Read); + } + do + { + ThdTask()->YieldToLowPQ(); + fail_point.MarkPauseReached(); + } while (fail_point.PauseRequested()); + }); } } while (must_wait()); const uint64_t waited_us = shard->DurationMicroseconds(start_us); @@ -312,8 +337,7 @@ IouringMgr::IouringMgr(const KvOptions *opts, uint32_t fd_limit) : AsyncIoManager(opts), fd_limit_(fd_limit) { memset(&ring_, 0, sizeof(ring_)); - const char *iostats_env = getenv("ELOQ_IO_STATS"); - io_stats_enabled_ = iostats_env != nullptr && iostats_env[0] == '1'; + io_stats_enabled_ = IoStatsEnabled(); lru_fd_head_.next_ = &lru_fd_tail_; lru_fd_tail_.prev_ = &lru_fd_head_; @@ -321,11 +345,13 @@ IouringMgr::IouringMgr(const KvOptions *opts, uint32_t fd_limit) write_req_pool_ = std::make_unique(pool_size); merged_write_req_pool_ = std::make_unique(pool_size); - // In-flight page-IO budgets (docs/design/io_qos.md M1/M2). The write cap - // shares max_inflight_write with the request-pool sizing above, so the - // pool bound and the QoS bound coincide by construction. The read budget - // carries the background sub-budget; the write budget has none — all - // page writes come from write tasks, which are background by definition. + // In-flight page-IO budgets (docs/design/io_qos.md M1/M2). The request + // pools above count request objects, while the write budget counts page + // units and permits one oversized request to run alone; sharing the option + // makes pool sizing conservative but does not make the bounds identical. + // The read budget carries the background sub-budget; the write budget has + // none — all page writes come from write tasks, which are background by + // definition. read_budget_.SetCap(options_->max_inflight_read); write_budget_.SetCap(options_->max_inflight_write); if (options_->max_inflight_read != 0) @@ -2250,6 +2276,7 @@ void IouringMgr::PollComplete() task = static_cast(ptr); if (type == UserDataType::KvTaskPageRead) { + TEST_FAIL_POINT_ACTION("KvTaskPageReadCqe", cqe->res = -EIO); read_budget_.Release(1, task->IsBackground()); if (io_stats_enabled_) { @@ -2265,6 +2292,7 @@ void IouringMgr::PollComplete() BaseReq *req = static_cast(ptr); if (type == UserDataType::BaseReqPageRead) { + TEST_FAIL_POINT_ACTION("BaseReqPageReadCqe", cqe->res = -EIO); read_budget_.Release(1, req->task_->IsBackground()); } req->res_ = cqe->res; diff --git a/src/kv_options.cpp b/src/kv_options.cpp index 1f6215fb4..08f43f5d4 100644 --- a/src/kv_options.cpp +++ b/src/kv_options.cpp @@ -151,8 +151,8 @@ int KvOptions::LoadFromIni(const char *path) } if (reader.HasValue(sec_run, "max_inflight_write")) { - max_inflight_write = - reader.GetUnsigned(sec_run, "max_inflight_write", 4096); + max_inflight_write = reader.GetUnsigned( + sec_run, "max_inflight_write", max_inflight_write); } if (reader.HasValue(sec_run, "max_inflight_read")) { diff --git a/tests/common.cpp b/tests/common.cpp index c786283c3..28839910a 100644 --- a/tests/common.cpp +++ b/tests/common.cpp @@ -16,15 +16,14 @@ eloqstore::EloqStore *InitStore(const eloqstore::KvOptions &opts, bool cleanup) // destructor's worker-thread joins and LRU-cached fd releases finish // before we count the new store's fd budget below. // - // Every test-created store must go through InitStore — never construct - // an EloqStore directly in a test while another may be running. The - // process-global `eloq_store` pointer (Options()/Comp() plumbing) - // assumes at most one started store per process; a second concurrent - // instance leaves one store's teardown reading a nulled/foreign global - // (observed as a flaky SIGSEGV in Prewarmer::Shutdown). Tests that need - // to preserve on-disk/cloud state across store generations (warm - // restart, cache-trim) pass cleanup = false instead of bypassing - // InitStore. + // Tests using this shared fixture must go through InitStore — never mix a + // directly-owned store with this process-global instance. The global + // Options()/Comp() plumbing assumes compatible live stores; overlapping + // incompatible instances can leave teardown reading a nulled/foreign + // global (observed as a flaky SIGSEGV in Prewarmer::Shutdown). Intentional + // multi-instance/topology tests own and coordinate all instances instead. + // Tests that only need to preserve on-disk/cloud state across generations + // (warm restart, cache-trim) pass cleanup = false. if (eloq_store) { if (!eloq_store->IsStopped()) diff --git a/tests/common.h b/tests/common.h index d158a38b6..363e5f7f3 100644 --- a/tests/common.h +++ b/tests/common.h @@ -82,9 +82,11 @@ const eloqstore::KvOptions cloud_archive_opts = { * Create (or replace) the per-binary test store. Stops and destroys any * previously created store first, so at most one EloqStore instance is ever * started in the process — required by the process-global Options() pointer. - * All test store creation must go through here; do not construct EloqStore - * directly in tests. Pass cleanup = false to keep existing local/cloud state - * (warm-restart and cache-reuse tests). + * Tests using this shared fixture must create stores through InitStore. + * Intentional multi-instance/topology tests may own EloqStore instances + * directly and are responsible for compatible process-global options. Pass + * cleanup = false to keep existing local/cloud state (warm-restart and + * cache-reuse tests). */ eloqstore::EloqStore *InitStore(const eloqstore::KvOptions &opts, bool cleanup = true); diff --git a/tests/eloq_store_test.cpp b/tests/eloq_store_test.cpp index 6466a149d..28a6dbf5c 100644 --- a/tests/eloq_store_test.cpp +++ b/tests/eloq_store_test.cpp @@ -14,6 +14,8 @@ namespace fs = std::filesystem; +static_assert(sizeof(eloqstore::IoQosStats) > 0); + eloqstore::KvOptions CreateValidOptions(const fs::path &test_dir) { eloqstore::KvOptions options; @@ -41,6 +43,40 @@ void CleanupTestDir(const fs::path &test_dir) fs::remove_all(test_dir); } } + +TEST_CASE("KvOptions parses QoS knobs and preserves malformed defaults", + "[eloq_store]") +{ + const fs::path test_dir = CreateTestDir("_qos_options"); + const fs::path ini_path = test_dir / "eloqstore.ini"; + { + std::ofstream ini(ini_path); + REQUIRE(ini.is_open()); + ini << "[run]\nmax_inflight_write = 73\n" + "max_inflight_read = 17\n" + "bg_read_ratio = 42\n" + "[permanent]\nstore_path = /tmp/unused\n"; + } + + eloqstore::KvOptions options; + REQUIRE(options.LoadFromIni(ini_path.c_str()) == 0); + REQUIRE(options.max_inflight_write == 73); + REQUIRE(options.max_inflight_read == 17); + REQUIRE(options.bg_read_ratio == 42); + + { + std::ofstream ini(ini_path); + REQUIRE(ini.is_open()); + ini << "[run]\nmax_inflight_write = invalid\n" + "[permanent]\nstore_path = /tmp/unused\n"; + } + options = eloqstore::KvOptions{}; + REQUIRE(options.LoadFromIni(ini_path.c_str()) == 0); + REQUIRE(options.max_inflight_write == + eloqstore::KvOptions{}.max_inflight_write); + CleanupTestDir(test_dir); +} + TEST_CASE("EloqStore ValidateOptions validates all parameters", "[eloq_store]") { auto test_dir = CreateTestDir("_validate_options"); @@ -49,6 +85,11 @@ TEST_CASE("EloqStore ValidateOptions validates all parameters", "[eloq_store]") // Test valid configuration REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == true); + // Write budget 0 is not a disable switch: every write must be bounded. + options.max_inflight_write = 0; + REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == false); + options = CreateValidOptions(test_dir); // restore valid value + // Test data_page_size that is not page-aligned options.data_page_size = 4097; // not page-aligned REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == false); diff --git a/tests/io_qos.cpp b/tests/io_qos.cpp index 3f21fff47..cb2711d41 100644 --- a/tests/io_qos.cpp +++ b/tests/io_qos.cpp @@ -6,8 +6,11 @@ * high-watermarks respect the caps (except the documented oversized-request * admission), and disabled budgets stay untouched. */ +#include + #include #include +#include #include "async_io_manager.h" #include "common.h" @@ -16,6 +19,11 @@ using test_util::MapVerifier; +namespace eloqstore +{ +DECLARE_uint64(max_processing_time_microseconds); +} + namespace { eloqstore::IoQosStats ShardStats(const eloqstore::EloqStore *store) @@ -52,6 +60,10 @@ TEST_CASE("io budgets: accounting invariants under tiny caps", "[io_qos]") REQUIRE(stats.read_.high_watermark_ <= 4); REQUIRE(stats.write_.high_watermark_ >= 1); REQUIRE(stats.write_.high_watermark_ <= 8); + REQUIRE(stats.read_.admitted_pages_ > 0); + REQUIRE(stats.write_.admitted_pages_ > 0); + REQUIRE(stats.fdatasync_count_ > 0); + REQUIRE(stats.fdatasync_us_ > 0); } TEST_CASE("io budgets: overflow read batch larger than the cap", "[io_qos]") @@ -79,6 +91,7 @@ TEST_CASE("io budgets: overflow read batch larger than the cap", "[io_qos]") REQUIRE(stats.read_.high_watermark_ <= 4); // A 128-page batch through a 4-page budget must have waited. REQUIRE(stats.read_.blocked_count_ > 0); + REQUIRE(stats.read_.blocked_us_ > 0); } TEST_CASE("io budgets: merged append writes and oversized admission", @@ -237,7 +250,7 @@ TEST_CASE("io budgets: negative WriteReq CQE drains and recovers", "[io_qos]") store->ExecSync(&failed); eloqstore::FailPoint::GetInstance().Disarm(); - REQUIRE(failed.Error() != eloqstore::KvError::NoError); + REQUIRE(failed.Error() == eloqstore::KvError::IoFail); REQUIRE(ShardStats(store).write_.inflight_ == 0); eloqstore::BatchWriteRequest recovery; @@ -270,7 +283,7 @@ TEST_CASE("io budgets: negative MergedWriteReq CQE drains and recovers", store->ExecSync(&failed); eloqstore::FailPoint::GetInstance().Disarm(); - REQUIRE(failed.Error() != eloqstore::KvError::NoError); + REQUIRE(failed.Error() == eloqstore::KvError::IoFail); REQUIRE(ShardStats(store).write_.inflight_ == 0); eloqstore::BatchWriteRequest recovery; @@ -281,6 +294,72 @@ TEST_CASE("io budgets: negative MergedWriteReq CQE drains and recovers", REQUIRE(ShardStats(store).write_.inflight_ == 0); } +TEST_CASE("io budgets: negative KvTaskPageRead CQE drains and recovers", + "[io_qos]") +{ + eloqstore::KvOptions opts = default_opts; + opts.max_inflight_read = 4; + eloqstore::EloqStore *store = InitStore(opts); + const eloqstore::TableIdent tbl_id{"qos-read-cqe", 0}; + MapVerifier seed(tbl_id, store, false); + seed.SetValueSize(200); + seed.Upsert(0, 100); + const std::string key = test_util::Key(7, 7); + const std::string expected = seed.DataSet().at(key).value_; + + eloqstore::ReadRequest failed; + failed.SetArgs(tbl_id, key); + eloqstore::FailPoint::GetInstance().ArmOnce("KvTaskPageReadCqe"); + store->ExecSync(&failed); + eloqstore::FailPoint::GetInstance().Disarm(); + + REQUIRE(failed.Error() == eloqstore::KvError::IoFail); + REQUIRE(ShardStats(store).read_.inflight_ == 0); + + eloqstore::ReadRequest recovery; + recovery.SetArgs(tbl_id, key); + store->ExecSync(&recovery); + REQUIRE(recovery.Error() == eloqstore::KvError::NoError); + REQUIRE(recovery.value_ == expected); + REQUIRE(ShardStats(store).read_.inflight_ == 0); +} + +TEST_CASE("io budgets: negative BaseReqPageRead CQE drains and recovers", + "[io_qos]") +{ + eloqstore::KvOptions opts = default_opts; + opts.max_inflight_read = 4; + opts.overflow_pointers = 128; + eloqstore::EloqStore *store = InitStore(opts); + const eloqstore::TableIdent tbl_id{"qos-batch-read-cqe", 0}; + MapVerifier seed(tbl_id, store, false); + constexpr uint32_t value_size = 32 * 1024; + seed.SetValueSize(value_size); + seed.Upsert(0); + const std::string key = test_util::Key(0, 7); + const std::string expected = seed.DataSet().at(key).value_; + + eloqstore::ReadRequest failed; + failed.SetArgs(tbl_id, key); + eloqstore::FailPoint::GetInstance().ArmOnce("BaseReqPageReadCqe"); + store->ExecSync(&failed); + eloqstore::FailPoint::GetInstance().Disarm(); + + REQUIRE(failed.Error() == eloqstore::KvError::IoFail); + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.read_.inflight_ == 0); + REQUIRE(stats.bg_read_.inflight_ == 0); + + eloqstore::ReadRequest recovery; + recovery.SetArgs(tbl_id, key); + store->ExecSync(&recovery); + REQUIRE(recovery.Error() == eloqstore::KvError::NoError); + REQUIRE(recovery.value_ == expected); + stats = ShardStats(store); + REQUIRE(stats.read_.inflight_ == 0); + REQUIRE(stats.bg_read_.inflight_ == 0); +} + TEST_CASE("io budgets: shutdown while tasks queue behind the budget", "[io_qos]") { @@ -346,14 +425,15 @@ TEST_CASE("io budgets: disabled read budget stays untouched", "[io_qos]") TEST_CASE("bg sub-budget: compaction batch reads are bounded", "[io_qos]") { - // Append mode with full-overwrite rounds drives space amplification past - // file_amplify_factor, so the shard schedules compaction between batch - // writes (per-table writes serialize behind the internal compact - // request, so by the time the last sync write returns, earlier - // compactions have completed). Compaction move batches issue up to - // 128-page ReadPages bursts from a BackgroundWrite task — the + // Append mode with repeated 3-of-5 partial overwrites drives space + // amplification past file_amplify_factor, so the shard schedules + // compaction between batch writes (per-table writes serialize behind the + // internal compact request, so by the time the last sync write returns, + // earlier compactions have completed). Compaction move batches issue up + // to 128-page ReadPages bursts from a BackgroundWrite task — the // BaseReqPageRead BG path — which must stay within the BG sub-budget - // (25% of 8 = 2 pages) while foreground keeps the full budget. + // (25% of 8 = 2 pages). Separate tests cover foreground capacity and + // concurrent foreground/background admission. eloqstore::KvOptions opts = append_opts; opts.file_amplify_factor = 2; opts.max_inflight_read = 8; @@ -388,6 +468,7 @@ TEST_CASE("bg sub-budget: compaction batch reads are bounded", "[io_qos]") REQUIRE(stats.write_.inflight_ == 0); // Compaction ran and its reads were charged to the BG class... REQUIRE(stats.bg_read_.high_watermark_ >= 1); + REQUIRE(stats.bg_read_.admitted_pages_ > 0); // ...and never exceeded the sub-budget. REQUIRE(stats.bg_read_.high_watermark_ <= 2); // A 128-page move batch through a 2-page sub-budget must have waited. @@ -443,12 +524,19 @@ TEST_CASE("bg sub-budget: batch-write leaf loads are background", "[io_qos]") REQUIRE(stats.bg_read_.high_watermark_ <= 2); } -TEST_CASE("bg sub-budget: pending demand survives the wake gap", "[io_qos]") +TEST_CASE("bg sub-budget: full reservation survives the wake gap", "[io_qos]") { + // Keep the scheduler in the high-priority loop after the fail point moves + // the woken BG task to low priority. This makes the intended ordering + // independent of the process-wide round-budget flag. + google::FlagSaver scheduler_flag_saver; + eloqstore::FLAGS_max_processing_time_microseconds = + std::numeric_limits::max(); + eloqstore::KvOptions opts = default_opts; opts.num_threads = 1; - opts.max_inflight_read = 2; - opts.bg_read_ratio = 50; // bg cap = 1 + opts.max_inflight_read = 1; + opts.bg_read_ratio = 100; // pending BG demand reserves the full cap opts.overflow_pointers = 128; eloqstore::EloqStore *store = InitStore(opts); @@ -464,8 +552,8 @@ TEST_CASE("bg sub-budget: pending demand survives the wake gap", "[io_qos]") std::atomic stop_fg{false}; std::atomic fg_failed{false}; + std::atomic fg_started{0}; std::atomic fg_done{0}; - eloqstore::FailPoint::GetInstance().ArmPersistent("IoBudgetBgWake"); std::vector readers; readers.reserve(8); @@ -474,6 +562,7 @@ TEST_CASE("bg sub-budget: pending demand survives the wake gap", "[io_qos]") readers.emplace_back( [&] { + fg_started.fetch_add(1, std::memory_order_relaxed); while (!stop_fg.load(std::memory_order_relaxed)) { eloqstore::ReadRequest req; @@ -487,7 +576,34 @@ TEST_CASE("bg sub-budget: pending demand survives the wake gap", "[io_qos]") } }); } - std::this_thread::sleep_for(std::chrono::milliseconds(20)); + + auto wait_until = [](auto &&condition, std::chrono::milliseconds timeout) + { + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (!condition()) + { + if (std::chrono::steady_clock::now() >= deadline) + { + return false; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return true; + }; + + eloqstore::IoQosStats primed_stats; + const bool fg_primed = wait_until( + [&] + { + primed_stats = ShardStats(store); + return fg_started.load(std::memory_order_relaxed) == + readers.size() && + fg_done.load(std::memory_order_relaxed) == 0 && + primed_stats.read_.inflight_ == opts.max_inflight_read && + primed_stats.read_.blocked_count_ >= + readers.size() - opts.max_inflight_read; + }, + std::chrono::seconds(2)); eloqstore::BatchWriteRequest bg_req; bg_req.SetTableId(bg_tbl); @@ -496,22 +612,40 @@ TEST_CASE("bg sub-budget: pending demand survives the wake gap", "[io_qos]") utils::UnixTs() + 1, eloqstore::WriteOp::Upsert); std::atomic bg_done{false}; + bool bg_issued = false; + if (fg_primed) + { + eloqstore::FailPoint::GetInstance().ArmPersistentPaused( + "IoBudgetBgWake"); + bg_issued = store->ExecAsyn( + &bg_req, + 0, + [&bg_done](eloqstore::KvRequest *) + { bg_done.store(true, std::memory_order_relaxed); }); + } + const bool wake_gap_observed = + bg_issued && + wait_until( + [&] { return eloqstore::FailPoint::GetInstance().PauseReached(); }, + std::chrono::seconds(2)); + const bool bg_incomplete_at_barrier = + !bg_done.load(std::memory_order_relaxed); + const eloqstore::IoQosStats barrier_stats = ShardStats(store); const uint64_t fg_before = fg_done.load(std::memory_order_relaxed); - store->ExecAsyn(&bg_req, - 0, - [&bg_done](eloqstore::KvRequest *) - { bg_done.store(true, std::memory_order_relaxed); }); - - const auto deadline = - std::chrono::steady_clock::now() + std::chrono::milliseconds(200); - while (std::chrono::steady_clock::now() < deadline) + eloqstore::FailPoint::GetInstance().ReleasePause(); + if (wake_gap_observed) { - if (bg_done.load(std::memory_order_relaxed) && - fg_done.load(std::memory_order_relaxed) > fg_before) + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::milliseconds(200); + while (std::chrono::steady_clock::now() < deadline) { - break; + if (bg_done.load(std::memory_order_relaxed) && + fg_done.load(std::memory_order_relaxed) > fg_before) + { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); } const bool bg_completed_while_armed = bg_done.load(std::memory_order_relaxed); @@ -526,7 +660,20 @@ TEST_CASE("bg sub-budget: pending demand survives the wake gap", "[io_qos]") store->Stop(); const eloqstore::IoQosStats stats = ShardStats(store); - CAPTURE(fg_before, fg_after); + CAPTURE(fg_primed, + bg_issued, + wake_gap_observed, + bg_incomplete_at_barrier, + barrier_stats.read_.inflight_, + primed_stats.read_.inflight_, + primed_stats.read_.blocked_count_, + fg_before, + fg_after); + REQUIRE(fg_primed); + REQUIRE(bg_issued); + REQUIRE(wake_gap_observed); + REQUIRE(bg_incomplete_at_barrier); + REQUIRE(barrier_stats.read_.inflight_ == 0); REQUIRE(fg_after > fg_before); REQUIRE(bg_completed_while_armed); REQUIRE_FALSE(fg_failed.load(std::memory_order_relaxed)); @@ -534,7 +681,7 @@ TEST_CASE("bg sub-budget: pending demand survives the wake gap", "[io_qos]") REQUIRE(bg_req.Error() == eloqstore::KvError::NoError); REQUIRE(stats.read_.inflight_ == 0); REQUIRE(stats.bg_read_.inflight_ == 0); - REQUIRE(stats.read_.high_watermark_ <= 2); + REQUIRE(stats.read_.high_watermark_ <= 1); REQUIRE(stats.bg_read_.high_watermark_ <= 1); REQUIRE(stats.bg_read_.blocked_count_ > 0); } From 23e1a1dee3df1743b7c09201c516b7a3f5e1a87e Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 17 Jul 2026 04:50:12 +0000 Subject: [PATCH 13/30] test(io): make wake-gap oracle architecture-independent --- tests/io_qos.cpp | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/tests/io_qos.cpp b/tests/io_qos.cpp index cb2711d41..a1dd9da3b 100644 --- a/tests/io_qos.cpp +++ b/tests/io_qos.cpp @@ -553,7 +553,6 @@ TEST_CASE("bg sub-budget: full reservation survives the wake gap", "[io_qos]") std::atomic stop_fg{false}; std::atomic fg_failed{false}; std::atomic fg_started{0}; - std::atomic fg_done{0}; std::vector readers; readers.reserve(8); @@ -572,7 +571,6 @@ TEST_CASE("bg sub-budget: full reservation survives the wake gap", "[io_qos]") { fg_failed.store(true, std::memory_order_relaxed); } - fg_done.fetch_add(1, std::memory_order_relaxed); } }); } @@ -598,7 +596,6 @@ TEST_CASE("bg sub-budget: full reservation survives the wake gap", "[io_qos]") primed_stats = ShardStats(store); return fg_started.load(std::memory_order_relaxed) == readers.size() && - fg_done.load(std::memory_order_relaxed) == 0 && primed_stats.read_.inflight_ == opts.max_inflight_read && primed_stats.read_.blocked_count_ >= readers.size() - opts.max_inflight_read; @@ -631,25 +628,10 @@ TEST_CASE("bg sub-budget: full reservation survives the wake gap", "[io_qos]") const bool bg_incomplete_at_barrier = !bg_done.load(std::memory_order_relaxed); const eloqstore::IoQosStats barrier_stats = ShardStats(store); - const uint64_t fg_before = fg_done.load(std::memory_order_relaxed); eloqstore::FailPoint::GetInstance().ReleasePause(); - if (wake_gap_observed) - { - const auto deadline = - std::chrono::steady_clock::now() + std::chrono::milliseconds(200); - while (std::chrono::steady_clock::now() < deadline) - { - if (bg_done.load(std::memory_order_relaxed) && - fg_done.load(std::memory_order_relaxed) > fg_before) - { - break; - } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - } const bool bg_completed_while_armed = - bg_done.load(std::memory_order_relaxed); - const uint64_t fg_after = fg_done.load(std::memory_order_relaxed); + wait_until([&] { return bg_done.load(std::memory_order_relaxed); }, + std::chrono::seconds(2)); stop_fg.store(true, std::memory_order_relaxed); eloqstore::FailPoint::GetInstance().Disarm(); @@ -666,15 +648,12 @@ TEST_CASE("bg sub-budget: full reservation survives the wake gap", "[io_qos]") bg_incomplete_at_barrier, barrier_stats.read_.inflight_, primed_stats.read_.inflight_, - primed_stats.read_.blocked_count_, - fg_before, - fg_after); + primed_stats.read_.blocked_count_); REQUIRE(fg_primed); REQUIRE(bg_issued); REQUIRE(wake_gap_observed); REQUIRE(bg_incomplete_at_barrier); REQUIRE(barrier_stats.read_.inflight_ == 0); - REQUIRE(fg_after > fg_before); REQUIRE(bg_completed_while_armed); REQUIRE_FALSE(fg_failed.load(std::memory_order_relaxed)); REQUIRE(bg_done.load(std::memory_order_relaxed)); From af86bf10aab79b396945caf95474095960212bca Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 17 Jul 2026 05:13:45 +0000 Subject: [PATCH 14/30] fix(benchmark): correct GET2 routing and percentiles --- benchmark/eloq_store_bm.cc | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/benchmark/eloq_store_bm.cc b/benchmark/eloq_store_bm.cc index 53878401c..e95daf3d8 100644 --- a/benchmark/eloq_store_bm.cc +++ b/benchmark/eloq_store_bm.cc @@ -576,6 +576,9 @@ void Benchmark::RunGet2(uint32_t client_threads, CHECK_GT(client_threads, 0U) << "GET2 client_threads must be positive"; CHECK_GT(inflight, 0U) << "GET2 inflight_per_client must be positive"; CHECK_GT(partition_count_, 0U) << "GET2 partition_count must be positive"; + CHECK(per_shard_cap == 0 || + (key_maximum_ >= key_minimum_ && + key_maximum_ - key_minimum_ >= partition_count_ - 1)); const uint16_t nshards = worker_cnt_; std::vector partition_shards(partition_count_); @@ -651,8 +654,26 @@ void Benchmark::RunGet2(uint32_t client_threads, } CHECK_LT(selected, partition_count_) << "GET2 per-shard cap accounting lost capacity"; + if (selected != part) + { + const uint32_t forward = + selected >= part + ? selected - part + : partition_count_ - (part - selected); + if (key_index + forward <= key_maximum_) + { + key_index += forward; + } + else + { + key_index -= partition_count_ - forward; + } + } part = selected; } + CHECK_GE(key_index, key_minimum_); + CHECK_LE(key_index, key_maximum_); + CHECK_EQ(key_index % partition_count_, part); op->shard_ = partition_shards[part]; op->key_.clear(); gen.generate_key(key_index, op->key_); @@ -741,8 +762,8 @@ void Benchmark::RunGet2(uint32_t client_threads, { return 0; } - size_t idx = - std::min(all.size() - 1, static_cast(p * all.size())); + const size_t idx = + static_cast(p * static_cast(all.size() - 1)); return all[idx]; }; LOG(INFO) << "GET2 finished: clients=" << client_threads From 17a8001b8f417f80b17e4f4a61f7a9386d092fe7 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 17 Jul 2026 05:49:22 +0000 Subject: [PATCH 15/30] fix(qos): apply remaining review cleanups --- docs/design/io_qos_impl_plan.md | 25 ++++++++++++++++--------- src/async_io_manager.cpp | 7 ++----- tests/eloq_store_test.cpp | 8 ++++---- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/docs/design/io_qos_impl_plan.md b/docs/design/io_qos_impl_plan.md index 1beb5655d..f82df71d8 100644 --- a/docs/design/io_qos_impl_plan.md +++ b/docs/design/io_qos_impl_plan.md @@ -118,7 +118,9 @@ policy in commit 1 is a plain `WakeN` on the released budget's zone. enforced; nothing else hard-fails. `max_inflight_read == 0` disables the read budget (documented). - `WriteReqPool` stays sized by `max_inflight_write` (unchanged line in - `IouringMgr` ctor) — pool bound and QoS bound coincide by construction. + `IouringMgr` ctor) as a conservative request-object allocation bound. QoS + counts configured page units, and a merged request may consume multiple + units. ### Stats @@ -317,14 +319,19 @@ caps to make blocking paths hot: ### Performance acceptance (per target device, after commit 3) -- **Success criterion** (from evaluation step 0 re-baseline): read p99 - during compaction ≤ agreed multiple of idle p99 (set the number from the - re-baseline gap, not a priori). -- **No-regression guards**: pure-read throughput and pure-write throughput - within noise (±3%) of pre-QoS baseline at default caps — the hot-path - cost is two counter checks per IO, so any regression indicates a wake - storm or false blocking. -- Sweeps: `max_inflight_read` ∈ {64, 128, 256, 512}, `bg_read_ratio` ∈ +- **User-confirmed product target**: read p99.9 below 10 ms during concurrent + write, compaction, and GC. +- **Measured status: FAIL / not release-ready.** Median read p99.9 was + 19.779 ms for the control and 18.711 ms for the candidate. This was a + same-binary comparison that differed only in the read budget; both conditions + retained the write budget, and the main campaign did not include a pure-write + comparison. +- **Outstanding no-regression guards**: pure-read throughput and pure-write + throughput within noise (±3%) of the pre-QoS baseline at default caps — the + hot-path cost is two counter checks per IO, so any regression indicates a + wake storm or false blocking. +- **Outstanding sweeps**: `max_inflight_read` ∈ {64, 128, 256, 512}, + `bg_read_ratio` ∈ {10, 25, 50}, `max_inflight_write` ∈ {256, 512, 1024}, `max_write_concurrency` ∈ {1, 2, 4, 8}; record read tails + BG throughput; pick defaults at the knees. diff --git a/src/async_io_manager.cpp b/src/async_io_manager.cpp index 5d6a03375..520802e4f 100644 --- a/src/async_io_manager.cpp +++ b/src/async_io_manager.cpp @@ -191,13 +191,10 @@ void IoBudget::Acquire(uint32_t cost, bool background) const uint32_t inflight = inflight_.load(std::memory_order_relaxed); if (use_bg) { - if (inflight + cost > cap_ && inflight != 0) - { - return true; - } const uint32_t bg_inflight = bg_inflight_.load(std::memory_order_relaxed); - return bg_inflight + cost > bg_cap_ && bg_inflight != 0; + return (inflight + cost > cap_ && inflight != 0) || + (bg_inflight + cost > bg_cap_ && bg_inflight != 0); } const uint32_t bg_inflight = bg_inflight_.load(std::memory_order_relaxed); diff --git a/tests/eloq_store_test.cpp b/tests/eloq_store_test.cpp index 28a6dbf5c..1167cff88 100644 --- a/tests/eloq_store_test.cpp +++ b/tests/eloq_store_test.cpp @@ -31,7 +31,8 @@ eloqstore::KvOptions CreateValidOptions(const fs::path &test_dir) fs::path CreateTestDir(const std::string &suffix = "") { - fs::path test_dir = fs::temp_directory_path() / ("eloqstore_test" + suffix); + fs::path test_dir = + fs::temp_directory_path() / "test-data" / ("eloqstore_test" + suffix); fs::create_directories(test_dir); return test_dir; } @@ -294,8 +295,7 @@ TEST_CASE("EloqStore Start validates local store paths", "[eloq_store]") } // test the non exist path - fs::path nonexistent_path = - fs::temp_directory_path() / "nonexistent_eloqstore_test"; + fs::path nonexistent_path = test_dir / "nonexistent_eloqstore_test"; options.store_path = {nonexistent_path}; { eloqstore::EloqStore store(options); @@ -307,7 +307,7 @@ TEST_CASE("EloqStore Start validates local store paths", "[eloq_store]") } // the path is file - fs::path file_path = fs::temp_directory_path() / "eloqstore_file_test"; + fs::path file_path = test_dir / "eloqstore_file_test"; std::ofstream file(file_path); file.close(); options.store_path = {file_path}; From 6f92c1e3592be1a22303f3cd64e59db973c2a8ed Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 17 Jul 2026 06:07:30 +0000 Subject: [PATCH 16/30] refactor(qos): simplify reviewed fixes --- benchmark/eloq_store_bm.cc | 9 +++------ src/async_io_manager.cpp | 6 ++---- tests/io_qos.cpp | 1 - 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/benchmark/eloq_store_bm.cc b/benchmark/eloq_store_bm.cc index e95daf3d8..bf5f533ff 100644 --- a/benchmark/eloq_store_bm.cc +++ b/benchmark/eloq_store_bm.cc @@ -660,13 +660,10 @@ void Benchmark::RunGet2(uint32_t client_threads, selected >= part ? selected - part : partition_count_ - (part - selected); - if (key_index + forward <= key_maximum_) + key_index += forward; + if (key_index > key_maximum_) { - key_index += forward; - } - else - { - key_index -= partition_count_ - forward; + key_index -= partition_count_; } } part = selected; diff --git a/src/async_io_manager.cpp b/src/async_io_manager.cpp index 520802e4f..bb8dd60d3 100644 --- a/src/async_io_manager.cpp +++ b/src/async_io_manager.cpp @@ -189,15 +189,13 @@ void IoBudget::Acquire(uint32_t cost, bool background) auto must_wait = [this, cost, use_bg]() { const uint32_t inflight = inflight_.load(std::memory_order_relaxed); + const uint32_t bg_inflight = + bg_inflight_.load(std::memory_order_relaxed); if (use_bg) { - const uint32_t bg_inflight = - bg_inflight_.load(std::memory_order_relaxed); return (inflight + cost > cap_ && inflight != 0) || (bg_inflight + cost > bg_cap_ && bg_inflight != 0); } - const uint32_t bg_inflight = - bg_inflight_.load(std::memory_order_relaxed); const uint32_t reserved = bg_pending_ != 0 && bg_inflight < bg_cap_ ? bg_cap_ - bg_inflight : 0; diff --git a/tests/io_qos.cpp b/tests/io_qos.cpp index a1dd9da3b..27707f61f 100644 --- a/tests/io_qos.cpp +++ b/tests/io_qos.cpp @@ -656,7 +656,6 @@ TEST_CASE("bg sub-budget: full reservation survives the wake gap", "[io_qos]") REQUIRE(barrier_stats.read_.inflight_ == 0); REQUIRE(bg_completed_while_armed); REQUIRE_FALSE(fg_failed.load(std::memory_order_relaxed)); - REQUIRE(bg_done.load(std::memory_order_relaxed)); REQUIRE(bg_req.Error() == eloqstore::KvError::NoError); REQUIRE(stats.read_.inflight_ == 0); REQUIRE(stats.bg_read_.inflight_ == 0); From e0f50ad3c0a45d070300913a4fa6928325d8781e Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 17 Jul 2026 06:21:28 +0000 Subject: [PATCH 17/30] fix(benchmark): validate GET2 key ranges --- benchmark/eloq_store_bm.cc | 5 +++-- docs/design/io_qos_impl_plan.md | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/benchmark/eloq_store_bm.cc b/benchmark/eloq_store_bm.cc index bf5f533ff..2a6e12acf 100644 --- a/benchmark/eloq_store_bm.cc +++ b/benchmark/eloq_store_bm.cc @@ -576,9 +576,10 @@ void Benchmark::RunGet2(uint32_t client_threads, CHECK_GT(client_threads, 0U) << "GET2 client_threads must be positive"; CHECK_GT(inflight, 0U) << "GET2 inflight_per_client must be positive"; CHECK_GT(partition_count_, 0U) << "GET2 partition_count must be positive"; + CHECK_GE(key_maximum_, key_minimum_) + << "GET2 key_maximum must not be less than key_minimum"; CHECK(per_shard_cap == 0 || - (key_maximum_ >= key_minimum_ && - key_maximum_ - key_minimum_ >= partition_count_ - 1)); + key_maximum_ - key_minimum_ >= partition_count_ - 1); const uint16_t nshards = worker_cnt_; std::vector partition_shards(partition_count_); diff --git a/docs/design/io_qos_impl_plan.md b/docs/design/io_qos_impl_plan.md index f82df71d8..dbfbdbe82 100644 --- a/docs/design/io_qos_impl_plan.md +++ b/docs/design/io_qos_impl_plan.md @@ -325,7 +325,9 @@ caps to make blocking paths hot: 19.779 ms for the control and 18.711 ms for the candidate. This was a same-binary comparison that differed only in the read budget; both conditions retained the write budget, and the main campaign did not include a pure-write - comparison. + comparison. These medians are from the 2026-07-16 local-NVMe campaign at + commit `c625004a32f474f446ba8adeba2d2d68f93dcee7` on `/dev/nvme1n1` + (`Microsoft NVMe Direct Disk v2`); they were not rerun at the final PR tip. - **Outstanding no-regression guards**: pure-read throughput and pure-write throughput within noise (±3%) of the pre-QoS baseline at default caps — the hot-path cost is two counter checks per IO, so any regression indicates a From 12855408ac1ae1fa1684b60b590d4d120702cea1 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 17 Jul 2026 06:27:30 +0000 Subject: [PATCH 18/30] refactor(benchmark): reuse GET2 routing offset --- benchmark/eloq_store_bm.cc | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/benchmark/eloq_store_bm.cc b/benchmark/eloq_store_bm.cc index 2a6e12acf..0d3278fd7 100644 --- a/benchmark/eloq_store_bm.cc +++ b/benchmark/eloq_store_bm.cc @@ -639,35 +639,29 @@ void Benchmark::RunGet2(uint32_t client_threads, uint32_t part = key_index % partition_count_; if (per_shard_cap > 0) { - uint32_t selected = partition_count_; - for (uint32_t offset = 0; offset < partition_count_; - ++offset) + uint32_t forward = 0; + for (; forward < partition_count_; ++forward) { const uint32_t candidate = - (static_cast(part) + offset) % + (static_cast(part) + forward) % partition_count_; if (shard_out[partition_shards[candidate]] < per_shard_cap) { - selected = candidate; + part = candidate; break; } } - CHECK_LT(selected, partition_count_) + CHECK_LT(forward, partition_count_) << "GET2 per-shard cap accounting lost capacity"; - if (selected != part) + if (forward != 0) { - const uint32_t forward = - selected >= part - ? selected - part - : partition_count_ - (part - selected); key_index += forward; if (key_index > key_maximum_) { key_index -= partition_count_; } } - part = selected; } CHECK_GE(key_index, key_minimum_); CHECK_LE(key_index, key_maximum_); From 0236e1c3fc99883e2a10f9a0dcece6fb9e76a5ed Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 17 Jul 2026 07:03:13 +0000 Subject: [PATCH 19/30] fix(rebase): drop obsolete task retry flags --- include/tasks/task.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/tasks/task.h b/include/tasks/task.h index dd19d8709..ad047602e 100644 --- a/include/tasks/task.h +++ b/include/tasks/task.h @@ -237,8 +237,6 @@ class KvTask // Stage-timing (ELOQ_IO_STATS=1): when this task began executing its // read; consumed (zeroed) by the first instrumented page read. uint64_t op_start_us_{0}; - bool needs_auto_reopen_{false}; - bool needs_oom_retry_{false}; TaskStatus status_{TaskStatus::Idle}; KvRequest *req_{nullptr}; From 591beaab0e124ee827ad98024a3fb430e0f1a536 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 17 Jul 2026 07:43:18 +0000 Subject: [PATCH 20/30] fix(qos): keep write cap opt-in by default --- docs/design/io_qos.md | 31 +++++++++++-------- docs/design/io_qos_impl_plan.md | 55 +++++++++++++++++++++------------ include/kv_options.h | 9 +++--- tests/eloq_store_test.cpp | 2 ++ tests/io_qos.cpp | 4 +-- 5 files changed, 63 insertions(+), 38 deletions(-) diff --git a/docs/design/io_qos.md b/docs/design/io_qos.md index 376229b5a..e0d556da0 100644 --- a/docs/design/io_qos.md +++ b/docs/design/io_qos.md @@ -80,10 +80,11 @@ implementation. subsequent foreground read — the yield discipline of #455 bounds CPU occupation per slice, not the IO submitted within it. -The current M1/M2 implementation uses per-shard defaults of 64 configured -data pages for reads, a 25% background read slice, and 512 configured data -pages for writes. The old `max_write_batch_pages` throttle has no effect; M3 -is still deferred. +The current M1/M2 implementation uses a per-shard default of 64 configured +data pages for reads and a 25% background read slice. The write budget is +available but retains its effectively-unbounded 32768-page default; 512 remains +an explicit opt-in value. The old `max_write_batch_pages` throttle has no +effect; M3 is still deferred. ## Device-Level Rationale @@ -246,17 +247,19 @@ compaction, since interference tracks bytes/sec rather than queue depth. ```cpp uint32_t max_inflight_read = 64; // configured data pages; 0 = disabled uint32_t bg_read_ratio = 25; // percent of max_inflight_read -uint32_t max_inflight_write = 512; // configured data pages; redefined option - // (was 32768, pool sizing only) +uint32_t max_inflight_write = 32768; // configured data pages; redefined option + // effectively unbounded by default uint64_t bg_write_rate_limit = 0; // bytes/sec; 0 = disabled (M3) ``` The WSL interference sweeps (2026-07-03) initially selected a read cap of 32 and `bg_read_ratio = 25`. Azure NVMe validation (2026-07-11) then showed real foreground queueing at 32 while 64–128 were indistinguishable, so the shipped -read default is 64 and the policy ratio remains 25%. The shipped write default -is 512. Real-device calibration (the QD sweep below) should still re-derive the -read cap per device; the ratio is policy and should transfer. +read default is 64 and the policy ratio remains 25%. A later Azure NVMe +acceptance run failed the p99.9 target with write cap 512, so the shipped write +default remains 32768 and smaller caps are opt-in. Real-device calibration (the +QD sweep below) should still re-derive the read cap per device; the ratio is +policy and should transfer. ### Sizing contract: device knob × policy knob @@ -317,7 +320,9 @@ The two read-side options deliberately live at different levels: small buffer pools must account for in-flight-write pins (production pools dwarf the ≤ max_inflight_write pages of pins; only tests noticed). - **`max_inflight_write` is redefined, not retired.** It is the M1 write cap - (configured data-page units, default dropped from 32768 to 512). + (configured data-page units). The default remains 32768, effectively + unbounded; 512 is an explicit opt-in value until the release acceptance + target is met. In-flight pages are bounded by `max(cap, one request's cost)` because one oversized request may run alone to guarantee progress. `WriteReqPool` stays numerically sized to the option, but counts request objects rather than page @@ -416,9 +421,9 @@ Export per-shard counters from day one; tuning must be measurement-driven: - Land M1+M2 with existing throttles left in place (loosened). - Re-run the read-vs-compaction benchmark; confirm the new mechanism alone protects foreground p99. - - Remove the `max_write_batch_pages` throttle and drop the - `max_inflight_write` default to its new QoS value in a follow-up - commit, so a regression identifies which mechanism was load-bearing. + - Remove the `max_write_batch_pages` throttle, but keep + `max_inflight_write` effectively unbounded by default until a calibrated + QoS value meets the release acceptance target. - Sweep `max_write_concurrency` ∈ {1, 2, 4, 8} in local mode; pick the throughput knee (expected at 2–4). - Evaluate M3 against the calibration curve; enable if M1+M2 leave a diff --git a/docs/design/io_qos_impl_plan.md b/docs/design/io_qos_impl_plan.md index dbfbdbe82..bb752b233 100644 --- a/docs/design/io_qos_impl_plan.md +++ b/docs/design/io_qos_impl_plan.md @@ -16,7 +16,7 @@ describe (CLAUDE.md requirement). | 1 | M1: budget state, acquire/release plumbing, options, stats getter | Core; behavior-neutral at default caps until tightened | | 2 | M2: `IsBackground()`, BG read sub-budget, wake policy | Small delta on top of 1 | | 3 | Interference benchmark + fio calibration script | Test-only | -| 4 | Retire `max_write_batch_pages` throttle; drop `max_inflight_write` default to QoS value | Behavioral; gated on benchmark results | +| 4 | Retire `max_write_batch_pages`; keep write QoS opt-in until acceptance passes | Behavioral; gated on benchmark results | | 5 | M3 rate limiter (deferred; only if step-4 benchmarks show a sustained-write gap) | Optional | ## Commit 1 — M1: in-flight page-IO budgets @@ -112,8 +112,8 @@ policy in commit 1 is a plain `WakeN` on the released budget's zone. - `kv_options.h`: add `max_inflight_read` (initially 256; shipped default 64); redefine `max_inflight_write` in configured data-page units (keep old default - 32768 in this commit — the shipped default drops to 512 in commit 4). Add INI - parsing in `kv_options.cpp` and equality-operator entries. + 32768 through release acceptance; lower caps remain opt-in). Add INI parsing + in `kv_options.cpp` and equality-operator entries. - `eloq_store.cpp` option validation: `max_inflight_write != 0` already enforced; nothing else hard-fails. `max_inflight_read == 0` disables the read budget (documented). @@ -202,16 +202,20 @@ it). ## Commit 4 — retire superseded throttles (gated on benchmarks) -> **Status: implemented** (2026-07-03), gated on the WSL interference -> campaign (real-device confirmation still advisable before release). +> **Status: implemented** (2026-07-03) except for default tightening. The +> throttle retirement shipped in the branch, but the later Azure NVMe +> acceptance run rejected making the calibrated write cap the default. > Details: `WritePage` throttle branch removed (budget admission comment > left in its place); `max_write_batch_pages` deprecated — parsed with a > LOG(WARNING), validation removed so all values are accepted, field documented > as no-effect; -> `max_inflight_write` default 32768 → 512, validated by a {512, 2048, -> 32768} sweep (512 binds marginally — hwm pinned, 6–26 blocks/30s — at -> equal-or-best write throughput and read tails; natural demand ceiling was -> 2048 = 8 concurrent 1MB merged writes). One behavioral consequence: with +> The WSL sweep initially selected `max_inflight_write = 512` from +> {512, 2048, 32768} (512 bound marginally — hwm pinned, +> 6–26 blocks/30s — at equal-or-best write throughput and read tails; +> natural demand ceiling was 2048 = 8 concurrent 1MB merged writes). +> The later Azure NVMe acceptance run did not justify making that value the +> default, so the final default remains 32768 and 512 is opt-in. One behavioral +> consequence of selecting a smaller value: with > enable_data_page_cache, write-promotion pins are now bounded by > max_inflight_write instead of the per-task drain; the data_page_cache OOM > test was recalibrated to `max_inflight_write = 1` (tiny 2-slot pool). @@ -224,9 +228,9 @@ that M1+M2 alone protect foreground p99 (design evaluation step 4): - Removed the `inflight_io_ >= max_write_batch_pages → WaitWrite()` branch in `WriteTask::WritePage` (`write_task.cpp` ~304). Keep the CPU yields and the terminal `WaitWrite()`. -- Dropped `max_inflight_write` default 32768 → calibrated value 512. - Release-notes entry: behavioral change for deployments setting it - explicitly. +- Kept `max_inflight_write` at 32768 by default; calibrated lower values such + as 512 are explicit opt-ins. Deployments setting the option must interpret + it under its new page-budget semantics. - Deprecated `max_write_batch_pages` in `kv_options.h` (keep parsing, ignore, log a warning) for one release before deletion. @@ -321,13 +325,26 @@ caps to make blocking paths hot: - **User-confirmed product target**: read p99.9 below 10 ms during concurrent write, compaction, and GC. -- **Measured status: FAIL / not release-ready.** Median read p99.9 was - 19.779 ms for the control and 18.711 ms for the candidate. This was a - same-binary comparison that differed only in the read budget; both conditions - retained the write budget, and the main campaign did not include a pure-write - comparison. These medians are from the 2026-07-16 local-NVMe campaign at - commit `c625004a32f474f446ba8adeba2d2d68f93dcee7` on `/dev/nvme1n1` - (`Microsoft NVMe Direct Disk v2`); they were not rerun at the final PR tip. +- **Measured status: FAIL / not release-ready.** + - In the 2026-07-16 same-binary campaign, median read p99.9 was + 19.779 ms with read QoS disabled and 18.711 ms with read QoS enabled. + Both conditions retained the 512-page write budget, so that campaign did + not isolate its effect. It ran at + `c625004a32f474f446ba8adeba2d2d68f93dcee7` on `/dev/nvme1n1` + (`Microsoft NVMe Direct Disk v2`). + - The 2026-07-17 three-condition follow-up used three balanced-order, + fresh-store repetitions at + `42b7f94084f196264615a77ccb20a35c284f5d12`. Median logical write + throughput was 19,473 key-ops/s with write cap 32768, 19,337 with write + cap 512 (-0.7%), and 19,693 with the full 64/25%/512 policy. The 512 cap + reached its high-water mark and recorded a median 53 write blocks, but was + not sustained-throughput-discriminating. Median read p99.9 was + 17.111/16.268/15.751 ms respectively: the full policy improved 7.9% versus + 32768 but still missed the 10 ms target by 57.5%. All nine runs had + concurrent compaction/GC and passed their validity gates. + - The tested configurations set their caps explicitly and ran with + `ELOQ_IO_STATS` disabled, so the subsequent default restoration and + opt-in timing cleanup do not change the exercised data path. - **Outstanding no-regression guards**: pure-read throughput and pure-write throughput within noise (±3%) of the pre-QoS baseline at default caps — the hot-path cost is two counter checks per IO, so any regression indicates a diff --git a/include/kv_options.h b/include/kv_options.h index 4ca19c71b..5dfea33df 100644 --- a/include/kv_options.h +++ b/include/kv_options.h @@ -84,11 +84,12 @@ struct KvOptions * Cannot be zero. * * NOTE: before the IO QoS work this option only sized the non-append - * write request pool and defaulted to 32768 (effectively unbounded); - * deployments that set it explicitly should re-derive their value as a - * queue-depth cap. + * write request pool. It retains the 32768 default (effectively + * unbounded) because the current in-flight budgets have not met the + * read-tail acceptance target. Deployments opting into write QoS should + * set a calibrated queue-depth cap explicitly. */ - uint32_t max_inflight_write = 512; + uint32_t max_inflight_write = 32 << 10; /** * @brief Per-shard cap on in-flight page-read IO, in configured * data-page units (`data_page_size`; docs/design/io_qos.md M1). Applies to diff --git a/tests/eloq_store_test.cpp b/tests/eloq_store_test.cpp index 1167cff88..cf649dd09 100644 --- a/tests/eloq_store_test.cpp +++ b/tests/eloq_store_test.cpp @@ -48,6 +48,8 @@ void CleanupTestDir(const fs::path &test_dir) TEST_CASE("KvOptions parses QoS knobs and preserves malformed defaults", "[eloq_store]") { + REQUIRE(eloqstore::KvOptions{}.max_inflight_write == 32768); + const fs::path test_dir = CreateTestDir("_qos_options"); const fs::path ini_path = test_dir / "eloqstore.ini"; { diff --git a/tests/io_qos.cpp b/tests/io_qos.cpp index 27707f61f..6312e4a40 100644 --- a/tests/io_qos.cpp +++ b/tests/io_qos.cpp @@ -418,7 +418,7 @@ TEST_CASE("io budgets: disabled read budget stays untouched", "[io_qos]") REQUIRE(stats.read_.inflight_ == 0); REQUIRE(stats.read_.high_watermark_ == 0); REQUIRE(stats.read_.blocked_count_ == 0); - // The write budget (default 512) still counts, it just never blocks. + // The effectively-unbounded default write budget still counts. REQUIRE(stats.write_.inflight_ == 0); REQUIRE(stats.write_.blocked_count_ == 0); } @@ -795,7 +795,7 @@ TEST_CASE("io qos stats: concurrent sampling", "[io_qos][stats]") TEST_CASE("io budgets: defaults are behavior-neutral", "[io_qos]") { - // At default caps (read 64, write 512) a single-threaded unit + // At default caps (read 64, write 32768) a single-threaded unit // workload of small values must never block on a budget: foreground // point reads are sequential (in-flight 1) and batch-write leaf loads // are sequential background singles, well under bg_cap = 16. From 49ec699cc512802c62b67aab4c5e1b1f0523e979 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 17 Jul 2026 07:43:33 +0000 Subject: [PATCH 21/30] fix(debug): refresh IO timing per request --- src/storage/shard.cpp | 4 ++++ src/tasks/read_task.cpp | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/storage/shard.cpp b/src/storage/shard.cpp index b78a2d6ec..821883ffa 100644 --- a/src/storage/shard.cpp +++ b/src/storage/shard.cpp @@ -1174,6 +1174,10 @@ void Shard::RetryOomRequest(KvRequest *req) #else req->done_.store(false, std::memory_order_relaxed); #endif + if (IoStatsEnabled()) + { + req->dbg_enqueue_us_ = ReadTimeMicroseconds(); + } // AddKvRequest refuses new work once the store is stopping; complete the // retried request with NotRunning instead of dropping it. if (!AddKvRequest(req)) diff --git a/src/tasks/read_task.cpp b/src/tasks/read_task.cpp index 7530c316e..4a3ba37cb 100644 --- a/src/tasks/read_task.cpp +++ b/src/tasks/read_task.cpp @@ -14,6 +14,16 @@ namespace eloqstore { namespace { +void BeginReadIoTiming() +{ + if (IoStatsEnabled()) + { + KvTask *task = ThdTask(); + task->op_cqe_us_ = 0; + task->op_start_us_ = Shard::ReadTimeMicroseconds(); + } +} + // Common path shared by all three Read() overloads: descend through the index // tree to the leaf data page that should contain @p search_key, load it, and // position @p iter at the matching entry. Returns KvError::NotFound when the @@ -35,10 +45,7 @@ KvError LocateAndProcess(const TableIdent &tbl_id, uint64_t &expire_ts, Handler &&handler) { - if (IoStatsEnabled()) - { - ThdTask()->op_start_us_ = Shard::ReadTimeMicroseconds(); - } + BeginReadIoTiming(); auto [root_handle, err] = shard->IndexManager()->FindRoot(tbl_id); CHECK_KV_ERR(err); RootMeta *meta = root_handle.Get(); @@ -202,6 +209,7 @@ KvError ReadTask::Floor(const TableIdent &tbl_id, uint64_t &expire_ts, IoStringBuffer *large_value) { + BeginReadIoTiming(); auto [root_handle, err] = shard->IndexManager()->FindRoot(tbl_id); CHECK_KV_ERR(err); RootMeta *meta = root_handle.Get(); From de1a8c6eb8d48d3cb9f717f36bfd11a66ee110e3 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 17 Jul 2026 08:09:46 +0000 Subject: [PATCH 22/30] docs(qos): align final review guidance --- db_stress/README.md | 4 ++-- db_stress/crash_test.py | 3 ++- db_stress/db_stress_gflags.cpp | 4 +++- docs/design/io_qos_impl_plan.md | 8 +++++--- include/eloq_store.h | 2 +- include/kv_options.h | 10 +++++----- 6 files changed, 18 insertions(+), 13 deletions(-) diff --git a/db_stress/README.md b/db_stress/README.md index e17832312..7bcc0a012 100644 --- a/db_stress/README.md +++ b/db_stress/README.md @@ -66,8 +66,8 @@ There are two groups of knobs: - `--data_append_mode`, `--pages_per_file_shift`, `--data_page_size` - `--buffer_pool_size`, `--io_queue_size`, `--fd_limit` - `--manifest_limit`, `--init_page_count` - - `--max_inflight_write`, `--max_write_batch_pages` + - `--max_inflight_write` (`--max_write_batch_pages` is deprecated and + ignored) - `--file_amplify_factor`, `--local_space_limit`, `--reserve_space_ratio` - `--overflow_pointers`, `--data_page_restart_interval`, `--index_page_restart_interval` - diff --git a/db_stress/crash_test.py b/db_stress/crash_test.py index ec317cbdc..cb4a04e6a 100755 --- a/db_stress/crash_test.py +++ b/db_stress/crash_test.py @@ -144,7 +144,8 @@ def randomize_dynamic_params(): "fd_limit": random.choice([5000, 10000, 15000]), "io_queue_size": random.choice([2048, 4096, 8192]), "max_inflight_write": random.choice([2048, 4096, 8192]), - "max_write_batch_pages": random.choice([32, 64, 128]), + # Deprecated and ignored; keep a fixed value for CLI compatibility. + "max_write_batch_pages": 64, # "coroutine_stack_size": lambda: random.choice([1<<13, 1<<14, 1<<15]), "file_amplify_factor": random.choice([2]), "reserve_space_ratio": random.choice([50, 100, 150, 200]), diff --git a/db_stress/db_stress_gflags.cpp b/db_stress/db_stress_gflags.cpp index 3c25d5bcc..d9858cef3 100644 --- a/db_stress/db_stress_gflags.cpp +++ b/db_stress/db_stress_gflags.cpp @@ -92,7 +92,9 @@ DEFINE_uint32(pages_per_file_shift, 11, "nums of filepage shift"); DEFINE_uint32(max_inflight_write, 4096, "Max amount of inflight write IO per thread"); -DEFINE_uint32(max_write_batch_pages, 64, "max pages per write batch"); +DEFINE_uint32(max_write_batch_pages, + 64, + "Deprecated compatibility option; ignored"); DEFINE_uint32(num_retained_archives, 0, "limit number of retained archives"); DEFINE_uint32(archive_interval_secs, 86400, "archive time interval in secs"); DEFINE_uint32(max_archive_tasks, 256, "max running archive tasks"); diff --git a/docs/design/io_qos_impl_plan.md b/docs/design/io_qos_impl_plan.md index bb752b233..ffeab5679 100644 --- a/docs/design/io_qos_impl_plan.md +++ b/docs/design/io_qos_impl_plan.md @@ -342,9 +342,11 @@ caps to make blocking paths hot: 17.111/16.268/15.751 ms respectively: the full policy improved 7.9% versus 32768 but still missed the 10 ms target by 57.5%. All nine runs had concurrent compaction/GC and passed their validity gates. - - The tested configurations set their caps explicitly and ran with - `ELOQ_IO_STATS` disabled, so the subsequent default restoration and - opt-in timing cleanup do not change the exercised data path. + - The tested configurations set their caps explicitly, so the subsequent + default restoration does not change the exercised admission + configuration. The retained run metadata did not record + `ELOQ_IO_STATS`; the later timing cleanup is diagnostic-only, but this + campaign was not repeated at that later commit. - **Outstanding no-regression guards**: pure-read throughput and pure-write throughput within noise (±3%) of the pre-QoS baseline at default caps — the hot-path cost is two counter checks per IO, so any regression indicates a diff --git a/include/eloq_store.h b/include/eloq_store.h index a83c412e2..221351c94 100644 --- a/include/eloq_store.h +++ b/include/eloq_store.h @@ -142,7 +142,7 @@ class KvRequest const TableIdent &TableId() const; uint64_t UserData() const; // Stage-timing instrumentation only (ELOQ_IO_STATS=1): microsecond - // timestamp when SendRequest enqueued this request to its shard. + // timestamp when the current attempt was enqueued to its shard. uint64_t dbg_enqueue_us_{0}; uint64_t dbg_dequeue_us_{0}; diff --git a/include/kv_options.h b/include/kv_options.h index 5dfea33df..14e42f2c6 100644 --- a/include/kv_options.h +++ b/include/kv_options.h @@ -83,11 +83,11 @@ struct KvOptions * exempt. * Cannot be zero. * - * NOTE: before the IO QoS work this option only sized the non-append - * write request pool. It retains the 32768 default (effectively - * unbounded) because the current in-flight budgets have not met the - * read-tail acceptance target. Deployments opting into write QoS should - * set a calibrated queue-depth cap explicitly. + * NOTE: before the IO QoS work this option only sized the write request + * pools. It retains the 32768 default (effectively unbounded) because the + * current in-flight budgets have not met the read-tail acceptance target. + * Deployments opting into write QoS should set a calibrated queue-depth + * cap explicitly. */ uint32_t max_inflight_write = 32 << 10; /** From f7a1552fb4566c9bf01b46bde0749e6f87e8367f Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Wed, 22 Jul 2026 20:30:50 -0700 Subject: [PATCH 23/30] fix(shard): TSC calibration must divide by measured elapsed time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit InitializeTscFrequency divided elapsed cycles by the REQUESTED 1ms sleep, but sleep_for reliably oversleeps by scheduler latency (~60us for a 1ms request). That inflated cycles-per-microsecond by ~6%, making every TSC-derived duration run ~6% slow: all timing gauges (budget blocked_us, IO stage timings) underreported by that factor, and the M4 rate budget's refill delivered exactly 94.1% of its configured rate — measured as a constant deficit across every load level, ratio, and budget until the cause was found. The systematic overshoot also defeats the calibration's stability check: consecutive measurements agree with each other while both being wrong. Divide by CLOCK_MONOTONIC-measured elapsed time instead. After the fix, delivered rate is within ~1% of configured. Co-Authored-By: Claude Fable 5 --- src/storage/shard.cpp | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/storage/shard.cpp b/src/storage/shard.cpp index 821883ffa..ae8b290c5 100644 --- a/src/storage/shard.cpp +++ b/src/storage/shard.cpp @@ -1511,13 +1511,30 @@ void Shard::InitializeTscFrequency() while (total_slept < MAX_TOTAL_MICROSECONDS) { + // Divide by the MEASURED elapsed wall time, not the + // requested sleep: sleep_for() reliably oversleeps by + // scheduler latency (~60us for a 1ms request), and dividing + // by the nominal duration inflated cycles-per-us by ~6% — + // making every TSC-derived duration (and the M4 rate + // budget's refill) run ~6% slow. The overshoot is + // systematic, so the stability check below cannot catch it. + timespec mono_start, mono_end; + clock_gettime(CLOCK_MONOTONIC, &mono_start); uint64_t start_cycles = __rdtsc(); std::this_thread::sleep_for( std::chrono::microseconds(SLEEP_MICROSECONDS)); uint64_t end_cycles = __rdtsc(); + clock_gettime(CLOCK_MONOTONIC, &mono_end); uint64_t elapsed_cycles = end_cycles - start_cycles; - uint64_t freq = elapsed_cycles / - SLEEP_MICROSECONDS; // cycles per microsecond + const int64_t elapsed_ns = + (mono_end.tv_sec - mono_start.tv_sec) * 1'000'000'000LL + + (mono_end.tv_nsec - mono_start.tv_nsec); + const uint64_t elapsed_us = + static_cast(std::max(elapsed_ns, 0)) / + 1000; + uint64_t freq = + elapsed_cycles / + std::max(elapsed_us, 1); // cycles per us total_slept += SLEEP_MICROSECONDS; From 47c2fa6dc78293b19ce41bb4c85289ac14fe2514 Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Wed, 22 Jul 2026 20:32:14 -0700 Subject: [PATCH 24/30] feat(io): replace count-based IO QoS with per-shard device rate limiting (M4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloud local NVMe is a provisioned rate limit, not a device: the test disk (Azure v2 direct disk) enforces ~275K IOPS and holds overflow IOs in quantized multi-ms delays (measured: read-only p99.9 = 3.3ms AT the ceiling with a bimodal 90us/3ms distribution and an empty middle; deep-queue fio reproduces the plateau at the same 275,328 ceiling, while sub-ceiling operation gives ~400us tails even under 400 MB/s of concurrent writes — the clean/throttled boundary sits at 100.5% of the fitted budget). Concurrency caps cannot express this: the count that holds the rate at the ceiling is rate x latency, a knife-edge that drifts with the workload mix. The engine must own the queue in the rate dimension so waiting happens in user space — FIFO, class-aware — instead of in the hypervisor's limiter. RateBudget: per-shard token buckets (ops + bytes) refilled lazily from the shard TSC clock once per event-loop iteration; debt admission (wait until positive, charge the full cost, let the balance go negative) makes the long-run rate exact and admits any single IO larger than the bucket without deadlock. The budget is PARTITIONED by class — foreground refills at (100 - rate_bg_ratio)%, background (background-task reads and all write-path IO) at rate_bg_ratio% — a shared balance was tried and rejected (one merged write's debit stalled every foreground read ~0.5ms/MB). Foreground alone may borrow background's surplus while background has no waiters and a positive balance, with the debit landing on the lender; symmetric borrowing was tried and reverted (storm-driven background skimmed ~2M ops/shard through microsecond foreground-idle windows: foreground 183K -> 116K QPS, p99.9 720us -> 5.6ms). Admission is peek-and-grant: waiters record their cost, the refill charges the FIFO head on its behalf and only then wakes it — exact wake counts for heterogeneous costs, no over-waking, no re-queue churn. New options: disk_rate_limit_iops (default ON at 275,000 per disk — a cloud-NVMe starting point; per-shard budget = iops x store paths / num_threads, multiple paths assumed identical devices; set ~95% of the fio-measured ceiling for precision; 0 disables), rate_bg_ratio (25), rate_limit_io_unit (2KB: the measured hypervisor accounting currency — a written 4KB costs two read units; >=16KB units measurably leak throttling under write load), rate_limit_burst_ms (2: the window only reshapes the latency distribution — smaller flattens median-up/ tail-down; 1/2/4ms cost no throughput), disk_rate_limit_mbps (0: only with a measured write-bandwidth ceiling), and max_inflight_io (0 = off: a single class-blind in-flight device-command window kept as a safety bound — measured inert on Azure, whose limiter charges rate, not instantaneous depth). The M1/M2 count budgets are retired: IoBudget is deleted, max_inflight_read and bg_read_ratio are parse-only deprecation no-ops, and max_inflight_write reverts to write request-pool sizing (32768). Under the partitioned rate budget the tuned count caps changed storm p99.9 by nothing measurable (709 vs 722us), and the write cap could never bind below one merged write buffer anyway. Validation (Azure Standard-L VM, single local NVMe, 4 shards, interference_bench, full ladders in docs/design/io_qos.md): read-only at QD32 budget sweep 200-260K all give p99.9 418-578us vs 3,392us uncapped (8.1x at 95% of ceiling, -11% QPS); with the storm, foreground reads hold ~205K QPS at p99.9 875us vs ~7ms unmanaged; at QD128 the unmanaged 6.9-7.7ms hypervisor plateau becomes an orderly ~2.6ms fair queue; rate_bg_ratio is exactly linear in both foreground QPS and background write MB/s at both depths with no floor down to 10%. docs/design/io_qos.md records the full design, calibration procedure, and the measured reasons each rejected alternative was rejected. Co-Authored-By: Claude Fable 5 --- benchmark/interference_bench.cpp | 93 ++-- docs/design/io_qos.md | 220 +++++++- include/async_io_manager.h | 240 ++++++--- include/kv_options.h | 140 ++++-- include/storage/shard.h | 2 +- include/tasks/task.h | 17 +- include/types.h | 29 +- src/async_io_manager.cpp | 464 ++++++++++------- src/kv_options.cpp | 47 +- src/storage/shard.cpp | 11 +- tests/io_qos.cpp | 829 ++++++++----------------------- 11 files changed, 1080 insertions(+), 1012 deletions(-) diff --git a/benchmark/interference_bench.cpp b/benchmark/interference_bench.cpp index be090d101..06259d78b 100644 --- a/benchmark/interference_bench.cpp +++ b/benchmark/interference_bench.cpp @@ -2,8 +2,9 @@ * IO QoS interference benchmark (docs/design/io_qos.md, plan commit 3). * * Measures how much a background write/compaction storm degrades foreground - * point-read tail latency, and how the IO QoS knobs (max_inflight_read, - * bg_read_ratio, max_inflight_write) change that. + * point-read tail latency, and how the IO QoS knobs (disk_rate_limit_iops, + * rate_bg_ratio, rate_limit_burst_ms, rate_limit_io_unit, max_inflight_io) + * change that. * * Phases: * 1. load — fill P partitions with K keys of ~val_size bytes each. @@ -30,11 +31,11 @@ * * Reports per phase: read QPS and exact p50/p90/p99/p99.9/max latency * (computed from raw samples, not a sliding window), the storm's write MB/s, - * and per-shard IoQosStats deltas (in-flight watermarks, budget-blocked - * counts/time, fdatasync). Greppable one-line summaries are prefixed with + * and per-shard IoQosStats deltas (rate-budget blocks/spend/borrows, io + * window, fdatasync). Greppable one-line summaries are prefixed with * "RESULT" for sweep scripts. A run exits nonzero if either measured phase * has errors, missing keys, fewer than --min_read_samples successes, or an - * enabled BG budget records no mixed-phase background page reads. + * enabled rate budget records no mixed-phase background spend. * * EloqStore options (including the QoS knobs) come from --kvoptions ini, so * sweeps only vary the ini / flags. See opts_interference.ini. @@ -435,42 +436,30 @@ void Load(eloqstore::EloqStore *store) void ReportQosDelta(const char *name, const eloqstore::IoQosStats &begin, const eloqstore::IoQosStats &end, - size_t shard, - uint32_t data_page_size, - double secs) + size_t shard) { auto d = [](uint64_t b, uint64_t e) { return e - b; }; - // Budgeted page IO in MB/s over the phase: user data, compaction - // relocations, and index pages. This is not total device traffic; metadata, - // manifest, bulk file/snapshot, fdatasync, and segment IO are unbudgeted. - auto mbps = [&](const eloqstore::IoQosStats::Budget &b, - const eloqstore::IoQosStats::Budget &e) - { - return secs > 0 ? (d(b.admitted_pages_, e.admitted_pages_) * - static_cast(data_page_size)) / - (secs * (1 << 20)) - : 0.0; - }; - LOG(INFO) << "RESULT qos phase=" << name << " shard=" << shard - << " read_hwm=" << end.read_.high_watermark_ << " read_blocked=" - << d(begin.read_.blocked_count_, end.read_.blocked_count_) - << " read_blocked_us=" - << d(begin.read_.blocked_us_, end.read_.blocked_us_) - << " read_budgeted_page_mbps=" << mbps(begin.read_, end.read_) - << " bg_read_hwm=" << end.bg_read_.high_watermark_ - << " bg_read_blocked=" - << d(begin.bg_read_.blocked_count_, end.bg_read_.blocked_count_) - << " bg_read_blocked_us=" - << d(begin.bg_read_.blocked_us_, end.bg_read_.blocked_us_) - << " bg_read_budgeted_page_mbps=" - << mbps(begin.bg_read_, end.bg_read_) - << " write_hwm=" << end.write_.high_watermark_ - << " write_blocked=" - << d(begin.write_.blocked_count_, end.write_.blocked_count_) - << " write_budgeted_page_mbps=" << mbps(begin.write_, end.write_) - << " fdatasync=" - << d(begin.fdatasync_count_, end.fdatasync_count_) - << " fdatasync_us=" << d(begin.fdatasync_us_, end.fdatasync_us_); + LOG(INFO) + << "RESULT qos phase=" << name << " shard=" << shard + << " fdatasync=" << d(begin.fdatasync_count_, end.fdatasync_count_) + << " fdatasync_us=" << d(begin.fdatasync_us_, end.fdatasync_us_) + << " rate_blocked=" + << d(begin.rate_.blocked_count_, end.rate_.blocked_count_) + << " rate_blocked_us=" + << d(begin.rate_.blocked_us_, end.rate_.blocked_us_) + << " rate_ops=" << d(begin.rate_.admitted_ops_, end.rate_.admitted_ops_) + << " rate_mb=" + << (d(begin.rate_.admitted_bytes_, end.rate_.admitted_bytes_) >> 20) + << " rate_borrowed=" + << d(begin.rate_.borrowed_ops_, end.rate_.borrowed_ops_) + << " bg_rate_blocked=" + << d(begin.bg_rate_.blocked_count_, end.bg_rate_.blocked_count_) + << " bg_rate_blocked_us=" + << d(begin.bg_rate_.blocked_us_, end.bg_rate_.blocked_us_) + << " bg_rate_borrowed=" + << d(begin.bg_rate_.borrowed_ops_, end.bg_rate_.borrowed_ops_) + << " io_hwm=" << end.io_window_hwm_ << " io_blocked=" + << d(begin.io_window_blocked_, end.io_window_blocked_); } } // namespace @@ -562,30 +551,20 @@ int main(int argc, char *argv[]) << static_cast(achieved_write_pct + 0.5); for (size_t s = 0; s < num_shards; s++) { - ReportQosDelta("baseline", - qos_start[s], - qos_mid[s], - s, - options.data_page_size, - FLAGS_baseline_secs); - ReportQosDelta("mixed", - qos_mid[s], - qos_end[s], - s, - options.data_page_size, - FLAGS_storm_secs); - mixed_bg_read_pages += qos_end[s].bg_read_.admitted_pages_ - - qos_mid[s].bg_read_.admitted_pages_; + ReportQosDelta("baseline", qos_start[s], qos_mid[s], s); + ReportQosDelta("mixed", qos_mid[s], qos_end[s], s); + mixed_bg_read_pages += qos_end[s].bg_rate_.admitted_ops_ - + qos_mid[s].bg_rate_.admitted_ops_; } const bool baseline_valid = ValidatePhase("baseline", baseline); const bool mixed_valid = ValidatePhase("mixed", storm_lat); bool interference_valid = true; - if (options.max_inflight_read != 0 && mixed_bg_read_pages == 0) + if (options.disk_rate_limit_iops != 0 && mixed_bg_read_pages == 0) { - LOG(ERROR) << "mixed phase produced no budgeted background page " - "reads; the intended write/compaction interference was " - "not exercised"; + LOG(ERROR) << "mixed phase recorded no background rate-budget " + "spend; the intended write/compaction interference " + "was not exercised"; interference_valid = false; } const bool valid = baseline_valid && mixed_valid && interference_valid; diff --git a/docs/design/io_qos.md b/docs/design/io_qos.md index e0d556da0..548e0e73d 100644 --- a/docs/design/io_qos.md +++ b/docs/design/io_qos.md @@ -118,6 +118,16 @@ Consequences for the design: ### M1: Per-shard in-flight page-IO caps, reads and writes separate +> **Superseded (2026-07-22).** The count budgets shipped, were validated, +> and were then retired in favor of M4: on rate-metered cloud disks the +> tail is set by the hypervisor's rate limiter, which a concurrency cap +> cannot address (measured: properly-sized count caps recovered a fraction +> of the rate budget's result), and the write cap could not bind below one +> merged write buffer. `max_inflight_read` / `bg_read_ratio` are deprecated +> no-ops; `max_inflight_write` reverted to write request-pool sizing; +> instantaneous depth bounding, if wanted, is the single class-blind +> `max_inflight_io` window. The sections below are kept as design record. + Two per-shard counters of in-flight **page** IO (in configured `data_page_size` units), with independent caps, enforced at the page-IO entry points of `IouringMgr`. @@ -173,6 +183,10 @@ Rules: ### M2: Foreground/background classification and BG sub-budget +> **Superseded (2026-07-22).** The FG/BG classification survives — it is +> the class model of the M4 rate budget (`rate_bg_ratio`) — but the +> count-domain sub-budget described here is retired with M1. + Every page IO is issued from a `KvTask` (`ThdTask()`), so classification is a task-type predicate. Add `KvTask::IsBackground()`: @@ -242,14 +256,208 @@ M3 is deferred until experiments confirm that in-flight capping alone (M1+M2) is insufficient — which the die-collision model predicts for sustained compaction, since interference tracks bytes/sec rather than queue depth. +**Status 2026-07-21: subsumed by M4 below.** The Azure campaign confirmed the +prediction — interference tracks rate, not queue depth — and additionally +showed that on cloud disks the dominant tail source is the *hypervisor's own +rate limiter*, which M4 addresses for all IO classes, not only background +writes. + +### M4: Per-shard device rate limiting (ops/sec + bytes/sec) + +**Motivation (measured 2026-07-21, Azure L-series local NVMe).** Cloud disks +are provisioned rate limits, not devices: `/dev/nvme1n1` enforced ~275K IOPS, +holding the overflow fraction of IOs for a quantized ~3 ms (bimodal latency: +90 µs or ~3 ms, nothing between). A read-only workload driving the disk at +its ceiling showed p99.9 = 3.3 ms with all engine stages clean; deep-queue +fio reproduced the plateau (6.7 ms) at the same 275K ceiling, and sub-ceiling +fio showed the true device tail is ~200 µs. AWS documents the same +enforcement for Nitro instance-store NVMe ("performance exceeded" counters); +GCP Local SSD limits scale with instance shape. Conclusion: **the engine must +own the queue** — admit IO below the provisioned ceiling so waiting happens +in user space (FIFO, foreground-prioritized, observable) instead of in the +hypervisor's limiter (random ~ms quantized holds, no priority). + +Count caps (M1) cannot express this robustly: the count that keeps the rate +at the ceiling is `rate × latency`, and the sweet spot is narrow — on the +test disk, 24 in-flight store-wide eliminated throttle holds (p99.9 +3,252 → 419 µs, −8% throughput) while 32 in-flight retained nearly the full +throttle tail and 16 forfeited 40% of throughput. A rate budget hits the +target directly and stays correct as latency shifts with the workload mix. + +**Budget derivation — simple division.** The deployment knows the per-disk +provisioned limits, the number of disks (`store_path` count), and the number +of shards (`num_threads`). Each shard's rate budget is: + +``` +shard_iops = disk_rate_limit_iops × num_store_paths / num_threads +shard_bytes = disk_rate_limit_mbps × num_store_paths / num_threads +``` + +This assumes shards spread IO uniformly across disks (true on average with +the store-path LUT). Instantaneous skew wastes some ceiling — accepted for +v1; per-(shard, path) buckets are a later refinement if skew shows up in the +gauges. + +**Mechanism.** Two token buckets per shard — `ops` and `bytes` — sharing the +M2 class policy: + +- *Lazy refill, no timers.* The shard event loop already runs continuously; + on each loop iteration (same hook as budget wake today), refill from the + TSC clock: `tokens += rate × (now − last)`, capped at + `rate × rate_limit_burst_ms`. `Shard::ReadTimeMicroseconds()` is already + static and cheap. +- *Debt admission.* `Acquire(cost)` waits until the balance is **positive**, + then subtracts the full cost, allowing the balance to go negative. Long-run + rate equals the refill rate exactly, a single large IO (1MB merged write = + 256 pages of byte-tokens) never deadlocks against a small bucket, and no + submission needs to be split. This also fixes the M1 write-cap quantum + problem measured on 2026-07-21: with count caps, `SubmitMergedWrite` + acquires whole-buffer units, so `max_inflight_write` below + `write_buffer_size / data_page_size` (256 at defaults) cannot bind — the + gauge showed 256 in flight under a cap of 64. Under M4 the byte bucket + charges actual bytes and paces exactly. +- *Class policy: partitioned buckets.* The budget is split by class: + foreground buckets refill at `(100 − rate_bg_ratio)` percent of the + shard rate and are charged only by foreground reads; background buckets + refill at `rate_bg_ratio` percent and are charged by background reads + and all write-path IO. (`rate_bg_ratio` is a separate option from + `bg_read_ratio` because it governs writes too, not only reads.) A + shared-balance variant — foreground draws freely, background capped at + a sub-share — was implemented first and measured worse (2026-07-21): + with writes charged at the device's true accounting granularity, one + merged write's debit drove the shared balance negative and every + foreground read arriving in the next ~0.5 ms/MB waited out the write's + debt (storm p99 1.0 ms shared vs 0.4 ms partitioned). Partitioning + isolates each class's debt at the cost of not lending an idle class's + headroom; **asymmetric borrow-when-idle** (implemented 2026-07-22) + repairs that: foreground only may admit on background's balance, while + background has no waiters and a positive balance, with the debit landing + on background. Read-only throughput recovers to the full configured rate + (243K vs 183K partition-only on the test disk, p99.9 413 µs) while storm + isolation is unchanged (183.5K, ~720 µs, writes pinned to the background + share). Background must never borrow: the symmetric variant was tried + and reverted — a closed-loop foreground's waiting zone empties for + microseconds between completion and resubmission, and in those windows + storm-driven background skimmed the foreground refill wholesale + (measured ~2M borrowed ops/shard per storm; foreground fell 183K → 116K + QPS and p99.9 720 µs → 5.6 ms). Foreground's share is a guarantee + against background and must hold regardless of how idle foreground + momentarily looks. Reverse lending (idle foreground donating to + background) would need genuine idle-hysteresis to be safe; deferred. + Each refill + wakes each class's zone independently — no cross-class wake coupling, + hence no starvation coupling either. +- *Charging points* are the existing M1 acquire sites: `ReadPage`/`ReadPages` + (ops = pages, bytes = pages × page size), `WritePage` (1 page), + `SubmitMergedWrite` (ops = ceil(len / rate_limit_io_unit) to mirror the + device-command split of large IOs, bytes = len), `fdatasync` (1 op). +- *Waiting/waking* reuse the existing per-class `WaitingZone`s; the wake site + moves from completion (`Release`) to refill (once per loop iteration). + There is no completion-driven release — spent tokens are gone; the refill + is the only credit source. + +**Relation to M1/M2.** The count caps remain as burst-depth guards (a rate +bucket alone would admit `burst_ms` worth of IO instantaneously after an idle +gap). Their sizing pressure disappears: set them to `2 × shard_iops × +t_read(loaded)` and forget them; the rate budget is the binding control. M3 +(background write bytes/sec) becomes the BG class share of the byte bucket — +no separate mechanism. + +**Recommended production configuration (validated 2026-07-22): the rate +knobs alone — the count caps are retired.** With the partitioned buckets, +the tuned count caps changed storm p99.9 by nothing measurable (709 vs +722 µs, same throughput), and they were subsequently removed. Configure +`disk_rate_limit_iops` (95% of the measured disk ceiling; default 275K — +the measured Azure v2 local-NVMe ceiling — with rate limiting ON by +default), `rate_bg_ratio` (the one policy choice), and +`rate_limit_io_unit` (a platform constant — 2 KB on Azure local NVMe, +i.e. written bytes cost twice read bytes per 4 KB; values ≥16 KB +measurably leak hypervisor throttling, 4 KB is equivalent to 2 KB). +`max_inflight_io` (single class-blind in-flight command window) exists as +an off-by-default safety bound; measured inert on Azure — the rate-metered +hypervisor does not penalize instantaneous depth at burst-window +magnitudes, so relocating the queue to user space changes nothing there. + +**New options.** + +``` +uint64_t disk_rate_limit_iops = 275000; // per store_path; 0 = off. ON by + // default: 275K is the measured Azure + // v2 local-NVMe ceiling and a sane + // cloud starting point. Multiple + // store paths are assumed identical + // devices; per-shard budget = + // iops x paths / num_threads. Devices + // faster than ~290K IOPS are capped + // until raised/disabled; measure and + // set 95% of ceiling for precision. +uint64_t disk_rate_limit_mbps = 0; // per store_path; 0 = bytes bucket off +uint32_t rate_limit_burst_ms = 2; // bucket capacity, ms of refill; + // = worst idle-edge latency transient. + // 1/2/4 ms cost no throughput (deep- + // queue A/B 2026-07-22); smaller + // flattens the distribution (median + // up, tail down), larger the reverse. + // 1 ms for tail-first deployments. +uint32_t rate_limit_io_unit = 2048; // ops-cost quantum for large IOs: + // the hypervisor ACCOUNTING currency + // (Azure: a written 4KB costs two + // read units, fio-fitted and + // ladder-confirmed; >=16KB leaks + // throttling). Overcharges writes on + // platforms with cheaper accounting + // — the safe direction. +uint32_t rate_bg_ratio = 25; // background share of the rate, percent +``` + +**Calibration.** Measure the disk ceiling once with deep-queue fio (4 jobs × +QD64 exposes the enforced rate directly as the IOPS plateau; the latency +plateau confirms throttling rather than saturation), then set +`disk_rate_limit_iops` to ~90–95% of it. The refill runs on the shard TSC +clock, so that clock's accuracy bounds the limiter's: a calibration bug +(fixed 2026-07-22 — cycles divided by the requested sleep instead of the +measured elapsed time) made the clock ~6% slow and every configured rate +silently deliver 94%; after the fix, delivered rate is within ~1% of +configured, which is also why the 5–10% headroom below the measured +ceiling matters — at 100% the margin is smaller than the ceiling's own +measurement error. On AWS, the documented +instance-store limits (and the NVMe "performance exceeded" counters) give +the number without probing. Leave 5–10% headroom: the hypervisor's own +bucket must never be the binding limiter, or its quantized holds reappear. + +**Validation plan.** + +- Read-only at the ceiling (the 2026-07-21 scenario): sweep + `disk_rate_limit_iops` ±20% around calibrated; expect a *wide* plateau of + good tails (vs. the count cap's knife-edge between 24 and 32 in-flight), + p99.9 within ~2× the true device tail, throughput within 10% of ceiling. +- Mixed storm: foreground p99.9 tracks the read-only number; background + (compaction) absorbs the deficit; write throughput cost reported. +- Cross-check the ops/bytes gauges against `iostat` per phase. + +**Future: adaptive rate.** The provisioned ceiling can be discovered at run +time: on AWS, read the exceeded-counters; elsewhere, detect the throttle +signature (bimodal completion-latency histogram with a fixed ~ms mode) and +walk the rate down until it disappears. Deferred until the static version is +validated in production. + ### New options (per shard) -```cpp -uint32_t max_inflight_read = 64; // configured data pages; 0 = disabled -uint32_t bg_read_ratio = 25; // percent of max_inflight_read -uint32_t max_inflight_write = 32768; // configured data pages; redefined option - // effectively unbounded by default -uint64_t bg_write_rate_limit = 0; // bytes/sec; 0 = disabled (M3) +> **Superseded (2026-07-22):** the surviving option surface is the M4 +> block below. `max_inflight_read` and `bg_read_ratio` are deprecated +> no-ops, `max_inflight_write` is write request-pool sizing again +> (default back to 32768), and `bg_write_rate_limit` (M3) was never +> shipped — background write pacing is the background share of the M4 +> byte bucket. + +``` +uint32_t max_inflight_read = 64; // configured data pages [DEPRECATED] +uint32_t bg_read_ratio = 25; // percent of max_inflight_read [DEPR.] +uint32_t max_inflight_write = 32768; // configured data pages; effectively + // unbounded by default + // [REVERTED to pool sizing] +uint64_t bg_write_rate_limit = 0; // bytes/sec; 0 = disabled (M3) [never + // shipped; subsumed by M4] ``` The WSL interference sweeps (2026-07-03) initially selected a read cap of 32 diff --git a/include/async_io_manager.h b/include/async_io_manager.h index bd0c194d5..179f7a59b 100644 --- a/include/async_io_manager.h +++ b/include/async_io_manager.h @@ -55,77 +55,148 @@ class ManifestFile }; /** - * @brief Per-shard in-flight page-IO budget (M1/M2 in docs/design/io_qos.md). + * @brief Per-shard device rate budget (docs/design/io_qos.md M4). * - * Counts admitted, not-yet-completed page IO in configured data-page units - * (`KvOptions::data_page_size`). Tasks block in Acquire when admission would - * exceed the cap; IouringMgr::PollComplete releases per CQE and wakes waiters, - * so release never depends on the blocked task being scheduled. + * Two token buckets — device operations and bytes — refilled lazily from + * the shard clock once per event-loop iteration (RefillAndWake, called + * from IouringMgr::Submit) and spent at the same entry points as the page + * IO paths. Admission uses debt semantics: a task waits until the + * balances are positive, then subtracts its full cost, letting a balance + * go negative. The long-run rate therefore equals the refill rate + * exactly, and a single IO costlier than the bucket capacity (a 1MB + * merged write against a small bytes bucket) admits without deadlock — + * it just drives the balance into debt that subsequent refills pay off + * before the next admission. * - * Optional background sub-budget (M2): when `bg_cap_` is non-zero, - * acquisitions with `background = true` are additionally bounded by - * `bg_inflight_ <= bg_cap_`. Background never exceeds its slice. Foreground - * may consume the entire budget while background has no pending demand; once - * a background acquisition enters the wait path, its unused entitlement - * (bg_cap_ - bg_inflight_) stays reserved through admission, including the - * wake-to-admit gap. Each class waits on its own FIFO zone; release wakes - * background waiters first and always wakes foreground, so neither sustained - * foreground saturation nor a saturated background queue can starve a class. + * The budget is PARTITIONED by class, not shared: foreground buckets + * refill at (100 - rate_bg_ratio) percent of the rate and are charged + * only by foreground IO; background buckets refill at rate_bg_ratio + * percent and are charged by background IO (background reads and all + * write-path IO). A shared balance was tried first and rejected: one + * large background debit (a merged write charged at the device's + * accounting granularity) drove the common balance negative and stalled + * every foreground read behind the write's debt for ~0.5 ms per MB + * (measured 2026-07-21, storm p99 1.0 ms shared vs 0.4 ms partitioned). + * Partitioning alone would waste idle-background headroom; ASYMMETRIC + * borrowing repairs that: FOREGROUND ONLY may admit on background's + * balance, and only while background has no waiters and a positive + * balance; the debit lands on background (see CanAdmit). With background + * idle, foreground runs at the full configured rate; the moment + * background demand appears, lending stops and background resumes at + * worst one IO-cost below zero. Background never borrows — foreground's + * share is a guarantee against background, and symmetric borrowing + * measurably collapsed that guarantee (see CanAdmit). + * There is deliberately no Release(): a rate budget meters issuance per + * second, not occupancy. A token is consumed by sending the IO, and how + * fast the device completes it is irrelevant — refunding on completion + * would collapse the rate limit into a mere concurrency ceiling. (The + * occupancy role belongs to the max_inflight_io window, which does pair + * AcquireIoWindow with a completion-time ReleaseIoWindow.) + * Wake-ups are refill-driven, never completion-driven: spent tokens are + * gone, time is the only credit source, so waiter progress depends only + * on the shard loop running. Each refill runs peek-and-grant per class + * (GrantWaiters): the waker charges the FIFO head's recorded cost on its + * behalf before waking it, so wake counts are exact even with + * heterogeneous costs and woken tasks never re-queue. * - * A cap of 0 disables the budget (Acquire/Release are no-ops). A request - * whose cost exceeds the (sub-)cap (e.g. a merged write larger than a small - * configured cap) is admitted alone once the relevant count drains to zero, - * so in-flight IO is bounded by max(cap, single-request cost) and progress - * is guaranteed. + * Balances are stored scaled by kScale (1 token = kScale units) so refill + * arithmetic (rate/sec x elapsed microseconds) stays in integers. */ -class IoBudget +class RateBudget { public: - void SetCap(uint32_t cap) + void SetRates(uint64_t ops_per_sec, + uint64_t bytes_per_sec, + uint32_t burst_ms, + uint32_t bg_ratio_pct); + bool Enabled() const { - cap_ = cap; + return fg_ops_rate_ != 0 || fg_bytes_rate_ != 0 || bg_ops_rate_ != 0 || + bg_bytes_rate_ != 0; } - void SetBgCap(uint32_t bg_cap) + void Acquire(uint32_t ops, uint64_t bytes, bool background); + void RefillAndWake(uint64_t now_us); + IoQosStats::Rate Stats() const { - bg_cap_ = bg_cap; - } - void Acquire(uint32_t cost, bool background = false); - void Release(uint32_t cost, bool background = false); - IoQosStats::Budget Stats() const - { - return {inflight_.load(std::memory_order_relaxed), - high_watermark_.load(std::memory_order_relaxed), - blocked_count_.load(std::memory_order_relaxed), + return {blocked_count_.load(std::memory_order_relaxed), blocked_us_.load(std::memory_order_relaxed), - admitted_pages_.load(std::memory_order_relaxed)}; + admitted_ops_.load(std::memory_order_relaxed), + admitted_bytes_.load(std::memory_order_relaxed), + borrowed_ops_.load(std::memory_order_relaxed)}; } - IoQosStats::Budget BgStats() const + IoQosStats::Rate BgStats() const { - return {bg_inflight_.load(std::memory_order_relaxed), - bg_high_watermark_.load(std::memory_order_relaxed), - bg_blocked_count_.load(std::memory_order_relaxed), + return {bg_blocked_count_.load(std::memory_order_relaxed), bg_blocked_us_.load(std::memory_order_relaxed), - bg_admitted_pages_.load(std::memory_order_relaxed)}; + bg_admitted_ops_.load(std::memory_order_relaxed), + bg_admitted_bytes_.load(std::memory_order_relaxed), + bg_borrowed_ops_.load(std::memory_order_relaxed)}; } private: - uint32_t cap_{0}; - std::atomic inflight_{0}; - uint32_t bg_cap_{0}; // 0 = no background sub-budget - std::atomic bg_inflight_{0}; - // BG acquisitions that entered the wait path but have not admitted yet. - // A wake does not end demand: the task can yield or re-wait before admit. - uint32_t bg_pending_{0}; - // The remaining atomic fields are observability only (tests, tuning, - // metrics); admission decisions additionally read bg_pending_. - std::atomic high_watermark_{0}; + static constexpr int64_t kScale = 1'000'000; + + /** + * @brief Whether the class's own balances are all positive. + */ + bool Positive(bool background) const; + /** + * @brief Admission test with asymmetric borrow-when-idle. + * + * A class is admissible on its own positive balances; foreground + * additionally on background's — while background has NO waiters (the + * instant its demand appears, lending stops) and a positive balance + * (surplus is lent, debt is never transferred). The borrow debit + * lands on background's buckets, so its refill repays the loan and it + * restarts at worst one IO-cost below zero. Background never borrows. + * + * @param background The class requesting admission. + * @return True if an acquisition of this class may proceed now. + */ + bool CanAdmit(bool background) const; + /** + * @brief Charge @p ops / @p bytes to the granting balances (own, or + * the lender's when borrowing) and update the class's counters. Debt + * semantics: balances may go negative. + */ + void Charge(uint32_t ops, uint64_t bytes, bool background); + /** + * @brief Peek-and-grant admission for the class's FIFO: while the + * class is admissible, charge the head waiter's recorded cost + * (KvTask::rate_wait_ops_/bytes_) on its behalf, then wake it. Exact + * wake counts for heterogeneous costs; the woken task's Acquire tail + * does nothing — this is the only waker of the rate zones, so a wake + * means the charge already happened. + */ + void GrantWaiters(bool background); + + // Per-class token rates per second; 0 disables that bucket. + uint64_t fg_ops_rate_{0}; + uint64_t fg_bytes_rate_{0}; + uint64_t bg_ops_rate_{0}; + uint64_t bg_bytes_rate_{0}; + uint64_t burst_us_{0}; // bucket capacity, microseconds of refill + uint64_t last_refill_us_{0}; // 0 = first refill pending + // Scaled balances (1 token = kScale units); may go negative (debt). + int64_t fg_ops_bal_{0}; + int64_t fg_bytes_bal_{0}; + int64_t bg_ops_bal_{0}; + int64_t bg_bytes_bal_{0}; + // Observability only; admission reads nothing but rates and balances. + // Atomic (relaxed) because IoQosStats is sampled from other threads + // while the shard runs (see the concurrent-sampling test); the shard + // thread is the only writer. std::atomic blocked_count_{0}; std::atomic blocked_us_{0}; - std::atomic admitted_pages_{0}; - std::atomic bg_high_watermark_{0}; + std::atomic admitted_ops_{0}; + std::atomic admitted_bytes_{0}; std::atomic bg_blocked_count_{0}; std::atomic bg_blocked_us_{0}; - std::atomic bg_admitted_pages_{0}; + std::atomic bg_admitted_ops_{0}; + std::atomic bg_admitted_bytes_{0}; + // Ops admitted from the other class's surplus (borrow-when-idle). + std::atomic borrowed_ops_{0}; + std::atomic bg_borrowed_ops_{0}; WaitingZone waiting_; // foreground waiters WaitingZone bg_waiting_; // background waiters }; @@ -627,12 +698,14 @@ class IouringMgr : public AsyncIoManager ~IouringMgr() override; KvError Init(Shard *shard) override; void Submit() override; - // A shard must not idle-block while this ring has prepared or - // submitted-but-unreaped IO: CQE delivery under DEFER_TASKRUN - // requires this thread to keep entering the kernel. The base class - // returns true unconditionally, which let shards sleep up to the - // 100ms request-wait timeout with CQEs pending (observed during - // prewarm, whose IO is not owned by an active task). + /** + * @brief A shard must not idle-block while this ring has prepared or + * submitted-but-unreaped IO: CQE delivery under DEFER_TASKRUN + * requires this thread to keep entering the kernel. The base class + * returns true unconditionally, which let shards sleep up to the + * 100ms request-wait timeout with CQEs pending (observed during + * prewarm, whose IO is not owned by an active task). + */ bool IsIdle() override { return inflight_ios_ == 0; @@ -1136,11 +1209,15 @@ class IouringMgr : public AsyncIoManager WaitingZone waiting_sqe_; uint32_t prepared_sqe_{0}; - // Per-shard in-flight page-IO budgets (M1, docs/design/io_qos.md). - // Reads and writes are separate device resources with independent caps - // (max_inflight_read / max_inflight_write). - IoBudget read_budget_; - IoBudget write_budget_; + RateBudget rate_budget_; + // Single class-blind in-flight device-command window (max_inflight_io; + // see AcquireIoWindow). 0 cap = disabled. The shard thread is the only + // writer; atomics (relaxed) allow cross-thread stats sampling. + uint32_t io_window_cap_{0}; + std::atomic io_window_inflight_{0}; + std::atomic io_window_hwm_{0}; + std::atomic io_window_blocked_{0}; + WaitingZone io_window_waiting_; // Write-path fdatasync instrumentation (FdatasyncFiles batches). std::atomic fdatasync_count_{0}; std::atomic fdatasync_us_{0}; @@ -1280,24 +1357,45 @@ class IouringMgr : public AsyncIoManager IoQosStats GetIoQosStats() const override { IoQosStats stats; - stats.read_ = read_budget_.Stats(); - stats.bg_read_ = read_budget_.BgStats(); - stats.write_ = write_budget_.Stats(); stats.fdatasync_count_ = fdatasync_count_.load(std::memory_order_relaxed); stats.fdatasync_us_ = fdatasync_us_.load(std::memory_order_relaxed); + stats.rate_ = rate_budget_.Stats(); + stats.bg_rate_ = rate_budget_.BgStats(); + stats.io_window_inflight_ = + io_window_inflight_.load(std::memory_order_relaxed); + stats.io_window_hwm_ = io_window_hwm_.load(std::memory_order_relaxed); + stats.io_window_blocked_ = + io_window_blocked_.load(std::memory_order_relaxed); return stats; } /** - * @brief Write-budget cost of a merged write in configured data-page units. - * Acquire (SubmitMergedWrite) and release (PollComplete) must use this - * same formula so the budget balances exactly. + * @brief Single class-blind cap on in-flight device commands + * (KvOptions::max_inflight_io; 0 = off). Companion to the rate budget: + * rate governs allocation per second with class policy; this bounds + * the instantaneous outstanding window toward the device, smoothing + * the rate bucket's burst release. Cost is device commands: 1 per page + * IO, ceil(len / kDeviceCmdBytes) per merged write. Oversized-request + * escape: a cost above the cap admits alone once the + * window drains. Completion-driven wake in PollComplete. + */ + void AcquireIoWindow(uint32_t cost); + void ReleaseIoWindow(uint32_t cost); + /** + * @brief Kernel splits large submissions into device commands of at + * most this size (queue-depth currency; distinct from + * rate_limit_io_unit, the hypervisor-accounting currency). + */ + static constexpr uint32_t kDeviceCmdBytes = 256 * 1024; + /** + * @brief In-flight-window cost of a submission of @p bytes, in device + * commands. Acquire and release must use the same formula. */ - uint32_t MergedWriteCost(size_t bytes) const + static uint32_t DeviceCmdCost(size_t bytes) { - const uint32_t page_size = options_->data_page_size; - return static_cast((bytes + page_size - 1) / page_size); + return static_cast((bytes + kDeviceCmdBytes - 1) / + kDeviceCmdBytes); } /** diff --git a/include/kv_options.h b/include/kv_options.h index 14e42f2c6..eb37c02b2 100644 --- a/include/kv_options.h +++ b/include/kv_options.h @@ -73,51 +73,105 @@ struct KvOptions */ uint32_t io_queue_size = 4096; /** - * @brief Per-shard cap on in-flight page-write IO, in configured - * data-page units (`data_page_size`; docs/design/io_qos.md M1). A merged - * append-mode write is charged by its byte length rounded up to a data - * page, so the cap means the same thing in append and non-append mode. - * Also sizes the write request pools. With `enable_data_page_cache`, it - * bounds cached-page write-promotion pins held until IO completion. - * Metadata, manifest, bulk file/snapshot, fdatasync, and segment IO are - * exempt. - * Cannot be zero. - * - * NOTE: before the IO QoS work this option only sized the write request - * pools. It retains the 32768 default (effectively unbounded) because the - * current in-flight budgets have not met the read-tail acceptance target. - * Deployments opting into write QoS should set a calibrated queue-depth - * cap explicitly. - */ - uint32_t max_inflight_write = 32 << 10; - /** - * @brief Per-shard cap on in-flight page-read IO, in configured - * data-page units (`data_page_size`; docs/design/io_qos.md M1). Applies to - * data-page reads (ReadPage/ReadPages). Local-GC ReadFile and - * prewarm/download whole-file bulk IO remain exempt, as do metadata, - * manifest, and segment IO. - * 0 disables the read budget. - * - * This is the device-calibration knob of the QoS sizing contract (see - * "Sizing contract" in docs/design/io_qos.md): size it near the - * device's bandwidth-delay product, c * max_random_read_IOPS * - * unloaded_read_latency / num_threads with c ~ 2-4. Foreground - * `read_blocked` staying ~0 under representative load validates the - * value; nonzero means undersized. - */ - uint32_t max_inflight_read = 64; - /** - * @brief Background share of max_inflight_read, in percent (clamped to - * 1..100; docs/design/io_qos.md M2). Page reads issued by batch-write and - * compaction tasks are bounded by this sub-budget so they cannot crowd out - * foreground point reads. Local GC and prewarm/download use exempt - * whole-file bulk IO instead. - * Foreground reads may use the entire read budget while no background - * acquisition is pending; pending background demand reserves its unused - * share through admission. No effect when the read budget is disabled - * (max_inflight_read = 0). + * @brief Write request-pool sizing (WriteReqPool / MergedWriteReqPool + * elements per shard). Cannot be zero. Historically this also acted as + * an in-flight page-write QoS cap (docs/design/io_qos.md M1); that + * role is retired — device admission control is the rate budget + * (disk_rate_limit_iops, M4) plus the optional class-blind window + * (max_inflight_io) — and the count-based cap could not bind below + * one merged write buffer anyway (256 pages at the 1MB default). + */ + uint32_t max_inflight_write = 32768; + /** + * @brief DEPRECATED — no effect. Formerly the per-shard in-flight + * page-read cap (docs/design/io_qos.md M1). Superseded by the device + * rate budget (disk_rate_limit_iops, M4), which controls the correct + * dimension on rate-metered disks, and by max_inflight_io for + * instantaneous depth bounding. Parsed with a warning for + * compatibility; scheduled for removal one release after deprecation. + */ + uint32_t max_inflight_read = 0; + /** + * @brief DEPRECATED — no effect. Formerly the background share of + * max_inflight_read (M2). Class policy lives in the rate budget's + * rate_bg_ratio, which covers background reads and all writes. + * Parsed with a warning for compatibility; scheduled for removal one + * release after deprecation. */ uint32_t bg_read_ratio = 25; + /** + * @brief Per-disk (per store_path) device rate limit in operations per + * second (docs/design/io_qos.md M4). 0 disables the ops bucket. Each + * shard's budget is disk_rate_limit_iops * store_path.size() / + * num_threads (multiple store paths are assumed to be identical + * devices). Set to ~90-95% of the disk's enforced ceiling (cloud disks + * are provisioned rate limits; measure with deep-queue fio or read the + * provider's documented figure) so that IO waits in the shard's + * admission queue — FIFO, foreground-first — instead of in the + * hypervisor's limiter, which holds overflow IOs in quantized multi-ms + * delays. The default is a reasonable starting point for current + * cloud-local NVMe (the measured Azure v2 direct disk ceiling); + * devices faster than ~290K IOPS are under-used until it is raised or + * disabled, and slower disks should be measured and set accordingly. + */ + uint64_t disk_rate_limit_iops = 275'000; + /** + * @brief Per-disk (per store_path) device rate limit in MB/s + * (docs/design/io_qos.md M4). 0 disables the bytes bucket. Divided + * across shards like disk_rate_limit_iops. + */ + uint64_t disk_rate_limit_mbps = 0; + /** + * @brief Rate-bucket capacity, in milliseconds of refill (M4). Bounds + * how much unspent credit can bank while a shard is idle and thus the + * largest instantaneous burst admitted after a gap; equivalently, the + * worst-case transient queueing an idle-to-busy edge adds. The window + * only reshapes the latency distribution (the mean is fixed by + * throughput): smaller = flatter (higher median, lower tail), larger = + * burstier (lower median, fatter tail). Measured 2026-07-22 at deep + * queue: 1/2/4 ms cost no throughput (within 0.5%); storm p99.9 was + * 2463/2619/2808 us and p50 382/243/133 us. Default 2 ms balances the + * two; use 1 ms when tails matter most. + */ + uint32_t rate_limit_burst_ms = 2; + /** + * @brief Ops-cost quantum for large IOs against the ops bucket (M4): + * a single submission of `len` bytes is charged + * ceil(len / rate_limit_io_unit) operations. This is the HYPERVISOR + * ACCOUNTING currency, not the kernel command split: on Azure local + * NVMe, written bytes cost two read units per 4KB (fio-fitted + * 2026-07-21/22, confirmed by the clean/throttled boundary and the + * engine currency ladder — 4KB units behaved identically to 2KB, + * >=16KB measurably re-exposed hypervisor throttling under write + * load). The 2KB default encodes that price. On platforms with + * different accounting it errs in the safe direction (overcharging + * writes paces background early instead of blowing the foreground + * tail); recalibrate with the fio boundary method in + * docs/design/io_qos.md if write throughput matters more. + */ + uint32_t rate_limit_io_unit = 2 * KB; + /** + * @brief Background share of the device rate budget, percent (M4). + * The rate budget is partitioned: background IO (background-task + * reads and all write-path IO) refills at this share, foreground + * reads at the remainder; covers writes as well, not only reads + * (unlike the retired count-era bg_read_ratio). Clamped to [1, 99]. + */ + uint32_t rate_bg_ratio = 25; + /** + * @brief Single class-blind cap on in-flight device commands per shard + * (M4 companion). 0 = off. Smooths the rate bucket's burst release + * toward the device: the rate budget governs allocation per second + * (with class policy), this bounds the instantaneous outstanding + * window. Deliberately does not distinguish reads/writes or + * foreground/background — by admission time the rate budget has + * already applied policy, and the device queue this replaces is + * class-blind anyway. Charged 1 per page read, ceil(len / 256KB) per + * merged write (the kernel's device-command split). Size a little + * above the throttled rate's bandwidth-delay product per shard + * (rate_per_shard x t_read, ~2x headroom). + */ + uint32_t max_inflight_io = 0; /** * @brief DEPRECATED — no effect. Formerly the per-write-task in-flight * page cap (the task drained to zero via WaitWrite once it had this diff --git a/include/storage/shard.h b/include/storage/shard.h index 5df5698c7..249f1cbaa 100644 --- a/include/storage/shard.h +++ b/include/storage/shard.h @@ -80,7 +80,7 @@ class Shard // Cheap TSC-based clock (rdtsc / calibrated cycles-per-us; ARM virtual // counter on aarch64). Public so shard-thread code outside Shard (e.g. - // IoBudget blocked-time accounting) can time intervals without a + // RateBudget blocked-time accounting) can time intervals without a // clock_gettime call. static uint64_t ReadTimeMicroseconds(); uint64_t DurationMicroseconds(uint64_t start_us); diff --git a/include/tasks/task.h b/include/tasks/task.h index ad047602e..ff4a9eb92 100644 --- a/include/tasks/task.h +++ b/include/tasks/task.h @@ -231,6 +231,11 @@ class KvTask int io_res_{0}; uint32_t io_flags_{0}; KvError result_err_{KvError::NoError}; + // Rate-budget waiter handshake (RateBudget::Acquire / RefillAndWake): + // the acquisition cost is recorded here at enqueue so the waker can + // peek the FIFO head and charge on its behalf before waking it. + uint32_t rate_wait_ops_{0}; + uint64_t rate_wait_bytes_{0}; // Loop-time (us) when this task's latest page-read CQE was reaped; // stage-timing instrumentation only (ELOQ_IO_STATS=1). uint64_t op_cqe_us_{0}; @@ -253,12 +258,20 @@ class WaitingZone /** * @brief Wake up to n waiters in FIFO order. * @return The number actually woken (< n when the zone drains), so the - * caller can forward unused wake credits to another zone (see - * IoBudget::Release). + * caller can forward unused wake credits to another zone. */ size_t WakeN(size_t n); void WakeAll(); bool Empty() const; + /** + * @brief The next waiter in FIFO order without waking it, or nullptr. + * Lets a waker peek the head's recorded cost and debit on its behalf + * before waking (see RateBudget::RefillAndWake). + */ + KvTask *Head() const + { + return head_; + } private: void PushBack(KvTask *task); diff --git a/include/types.h b/include/types.h index 9e11baaa2..7e28f4511 100644 --- a/include/types.h +++ b/include/types.h @@ -36,24 +36,25 @@ enum class StoreMode */ struct IoQosStats { - struct Budget + // Device rate budget (M4). rate_ is the foreground class; bg_rate_ is + // background (background-task reads and all write-path IO), paced by + // its partitioned share. + struct Rate { - uint32_t inflight_{0}; // pages currently admitted - uint32_t high_watermark_{0}; // max pages ever admitted uint64_t blocked_count_{0}; // acquisitions that had to wait uint64_t blocked_us_{0}; // cumulative wait time - // Cumulative configured data pages ever admitted by this budget. This - // is budgeted page-IO volume, not total device traffic. Metadata, - // manifest, bulk file/snapshot, fdatasync, and segment IO are - // unbudgeted and therefore absent. - uint64_t admitted_pages_{0}; + uint64_t admitted_ops_{0}; // cumulative device-op cost admitted + uint64_t admitted_bytes_{0}; // cumulative bytes admitted + // Of admitted_ops_, the ops granted from the OTHER class's idle + // surplus (borrow-when-idle; see RateBudget). + uint64_t borrowed_ops_{0}; }; - Budget read_; - // Background slice of read_ (M2), bounded by the bg sub-budget. Inflight, - // high-watermark, and admitted pages are subsets of read_; blocked fields - // are per-class (read_ is foreground, bg_read_ is background). - Budget bg_read_; - Budget write_; + Rate rate_; + Rate bg_rate_; + // Single class-blind in-flight device-command window (max_inflight_io). + uint32_t io_window_inflight_{0}; + uint32_t io_window_hwm_{0}; + uint64_t io_window_blocked_{0}; uint64_t fdatasync_count_{0}; // write-path fdatasync ops (FdatasyncFiles) uint64_t fdatasync_us_{0}; // cumulative batch wall time }; diff --git a/src/async_io_manager.cpp b/src/async_io_manager.cpp index bb8dd60d3..522528425 100644 --- a/src/async_io_manager.cpp +++ b/src/async_io_manager.cpp @@ -167,165 +167,228 @@ bool AsyncIoManager::IsIdle() return true; } -void IoBudget::Acquire(uint32_t cost, bool background) +void RateBudget::SetRates(uint64_t ops_per_sec, + uint64_t bytes_per_sec, + uint32_t burst_ms, + uint32_t bg_ratio_pct) { - if (cap_ == 0) + // Partition the total rate between the classes (see class comment): + // background gets ratio percent, foreground the rest. Clamped so both + // classes always have a nonzero share when the budget is enabled. + const uint32_t ratio = std::clamp(bg_ratio_pct, 1, 99); + fg_ops_rate_ = ops_per_sec * (100 - ratio) / 100; + fg_bytes_rate_ = bytes_per_sec * (100 - ratio) / 100; + bg_ops_rate_ = ops_per_sec * ratio / 100; + bg_bytes_rate_ = bytes_per_sec * ratio / 100; + burst_us_ = uint64_t{std::max(burst_ms, 1)} * 1000; +} + +bool RateBudget::Positive(bool background) const +{ + if (background) + { + return (bg_ops_rate_ == 0 || bg_ops_bal_ > 0) && + (bg_bytes_rate_ == 0 || bg_bytes_bal_ > 0); + } + return (fg_ops_rate_ == 0 || fg_ops_bal_ > 0) && + (fg_bytes_rate_ == 0 || fg_bytes_bal_ > 0); +} + +bool RateBudget::CanAdmit(bool background) const +{ + if (Positive(background)) + { + return true; + } + if (background) + { + // Background never borrows. Symmetric borrowing was tried and + // reverted (2026-07-22): a closed-loop foreground's waiting zone + // empties for microseconds between completion and resubmission, + // and in those windows storm-driven background (whose demand is + // effectively unbounded) skimmed the foreground refill wholesale — + // measured as bg_rate_borrowed ~2M ops/shard per storm, foreground + // QPS 183K -> 116K, p99.9 720 µs -> 5.6 ms. Foreground's share is + // a guarantee AGAINST background; it must hold no matter how idle + // foreground momentarily looks. + return false; + } + // Foreground may borrow background's surplus while background has no + // queued demand. Requiring a positive lender balance means surplus is + // lent, never debt — background restarts at worst one IO-cost below + // zero when its demand returns. Lending in this direction can only + // reduce device contention for the class the tail contract protects. + return bg_waiting_.Empty() && Positive(true); +} + +void IouringMgr::AcquireIoWindow(uint32_t cost) +{ + if (io_window_cap_ == 0) { return; } - const bool use_bg = background && bg_cap_ != 0; - // Admission conditions. `inflight != 0` implements the oversized-request - // escape per class: a request with cost > (sub-)cap is admitted alone - // once the relevant count drains, guaranteeing progress (see IoBudget - // doc comment). Background is additionally bounded by its sub-budget. - // Foreground must not take units reserved for pending background demand. - // Pending spans the full first-wait-to-admission interval, including a - // wake followed by a yield or re-wait, when the wait queue is empty. - // Without that reservation, sustained foreground saturation would - // starve background forever — every freed unit would be re-acquired by - // a foreground waiter and compaction could never bootstrap its share. - // While background has no pending demand, foreground may use the entire - // budget. - auto must_wait = [this, cost, use_bg]() - { - const uint32_t inflight = inflight_.load(std::memory_order_relaxed); - const uint32_t bg_inflight = - bg_inflight_.load(std::memory_order_relaxed); - if (use_bg) - { - return (inflight + cost > cap_ && inflight != 0) || - (bg_inflight + cost > bg_cap_ && bg_inflight != 0); - } - const uint32_t reserved = bg_pending_ != 0 && bg_inflight < bg_cap_ - ? bg_cap_ - bg_inflight - : 0; - const uint32_t fg_cap = cap_ > reserved ? cap_ - reserved : 0; - const bool exceeds_fg_cap = - uint64_t{inflight} + cost > uint64_t{fg_cap}; - // Oversized requests may exceed the configured cap only when they are - // alone and no background entitlement is reserved. Without the - // reserved check, a foreground request can steal a 100% BG slice in - // the wake-to-admit gap whenever total inflight briefly reaches zero. - const bool oversized_alone = - inflight == 0 && reserved == 0 && cost > cap_; - return exceeds_fg_cap && !oversized_alone; - }; - // Each class queues behind its own existing waiters (approximate FIFO - // across arrivals within the class). - WaitingZone &zone = use_bg ? bg_waiting_ : waiting_; - bool pending_bg = false; - if (!zone.Empty() || must_wait()) - { - // TSC-based clock (see Shard::ReadTimeMicroseconds): Acquire always - // runs on the shard thread, after Shard::Init calibrated the TSC. - const uint64_t start_us = shard->ReadTimeMicroseconds(); - std::atomic &blocked_count = - use_bg ? bg_blocked_count_ : blocked_count_; - blocked_count.store(blocked_count.load(std::memory_order_relaxed) + 1, - std::memory_order_relaxed); - if (use_bg) - { - ++bg_pending_; - pending_bg = true; - } + // Class-blind FIFO with the oversized-request escape (cost above the + // cap admits alone once the window drains). + // Queue behind existing waiters so arrival order is preserved. + auto inflight = [this] + { return io_window_inflight_.load(std::memory_order_relaxed); }; + if (!io_window_waiting_.Empty() || + (inflight() + cost > io_window_cap_ && inflight() != 0)) + { + io_window_blocked_.fetch_add(1, std::memory_order_relaxed); do { - zone.Wait(ThdTask()); - if (use_bg) - { - TEST_FAIL_POINT_ACTION("IoBudgetBgWake", { - FailPoint &fail_point = FailPoint::GetInstance(); - if (fail_point.PauseRequested() && - !fail_point.PauseReached()) - { - // Prove that the paused regression reached the exact - // wake gap with foreground demand both ready and - // queued, rather than relying on cumulative counters. - CHECK(!waiting_.Empty()); - CHECK_GT(shard->ready_tasks_.Size(), 0); - CHECK(shard->ready_tasks_.Peek()->Type() == - TaskType::Read); - } - do - { - ThdTask()->YieldToLowPQ(); - fail_point.MarkPauseReached(); - } while (fail_point.PauseRequested()); - }); - } - } while (must_wait()); - const uint64_t waited_us = shard->DurationMicroseconds(start_us); - std::atomic &blocked_us = - use_bg ? bg_blocked_us_ : blocked_us_; - blocked_us.store(blocked_us.load(std::memory_order_relaxed) + waited_us, - std::memory_order_relaxed); - } - if (pending_bg) - { - CHECK_GT(bg_pending_, 0); - --bg_pending_; - } - const uint32_t inflight = inflight_.load(std::memory_order_relaxed) + cost; - inflight_.store(inflight, std::memory_order_relaxed); - admitted_pages_.store( - admitted_pages_.load(std::memory_order_relaxed) + cost, - std::memory_order_relaxed); - if (inflight > high_watermark_.load(std::memory_order_relaxed)) + io_window_waiting_.Wait(ThdTask()); + } while (inflight() + cost > io_window_cap_ && inflight() != 0); + } + const uint32_t now_inflight = inflight() + cost; + io_window_inflight_.store(now_inflight, std::memory_order_relaxed); + if (now_inflight > io_window_hwm_.load(std::memory_order_relaxed)) { - high_watermark_.store(inflight, std::memory_order_relaxed); + io_window_hwm_.store(now_inflight, std::memory_order_relaxed); } - if (use_bg) +} + +void IouringMgr::ReleaseIoWindow(uint32_t cost) +{ + if (io_window_cap_ == 0) { - const uint32_t bg_inflight = - bg_inflight_.load(std::memory_order_relaxed) + cost; - bg_inflight_.store(bg_inflight, std::memory_order_relaxed); - bg_admitted_pages_.store( - bg_admitted_pages_.load(std::memory_order_relaxed) + cost, - std::memory_order_relaxed); - if (bg_inflight > bg_high_watermark_.load(std::memory_order_relaxed)) - { - bg_high_watermark_.store(bg_inflight, std::memory_order_relaxed); - } + return; + } + const uint32_t cur = io_window_inflight_.load(std::memory_order_relaxed); + assert(cur >= cost); + io_window_inflight_.store(cur - cost, std::memory_order_relaxed); + // Each freed command can admit at most one waiter; over-waking is safe + // (woken tasks re-check and re-wait). + io_window_waiting_.WakeN(cost); +} + +void RateBudget::Charge(uint32_t ops, uint64_t bytes, bool background) +{ + // Debt semantics: subtract the full cost, letting the balance go + // negative. Callers guarantee CanAdmit(background) held at the moment + // of the charge. + const int64_t ops_cost = int64_t{ops} * kScale; + const int64_t bytes_cost = static_cast(bytes) * kScale; + if (background) + { + // Background admits only on its own positive balances (CanAdmit + // never lets it borrow), so a background charge always debits the + // background buckets — the foreground debit is unreachable by + // construction, not by luck. + assert(Positive(true)); + bg_ops_bal_ -= ops_cost; + bg_bytes_bal_ -= bytes_cost; + bg_admitted_ops_.fetch_add(ops, std::memory_order_relaxed); + bg_admitted_bytes_.fetch_add(bytes, std::memory_order_relaxed); + return; + } + // Foreground: exhausted own balances mean this admission was granted + // from background's idle surplus (CanAdmit's borrow arm) — the debit + // lands on the lender so its refill repays the loan. + const bool borrowed = !Positive(false); + if (borrowed) + { + bg_ops_bal_ -= ops_cost; + bg_bytes_bal_ -= bytes_cost; + } + else + { + fg_ops_bal_ -= ops_cost; + fg_bytes_bal_ -= bytes_cost; + } + admitted_ops_.fetch_add(ops, std::memory_order_relaxed); + admitted_bytes_.fetch_add(bytes, std::memory_order_relaxed); + if (borrowed) + { + borrowed_ops_.fetch_add(ops, std::memory_order_relaxed); } } -void IoBudget::Release(uint32_t cost, bool background) +void RateBudget::Acquire(uint32_t ops, uint64_t bytes, bool background) { - if (cap_ == 0) + if (!Enabled()) { return; } - const uint32_t inflight = inflight_.load(std::memory_order_relaxed); - CHECK_GE(inflight, cost); - inflight_.store(inflight - cost, std::memory_order_relaxed); - if (background && bg_cap_ != 0) - { - const uint32_t bg_inflight = - bg_inflight_.load(std::memory_order_relaxed); - CHECK_GE(bg_inflight, cost); - bg_inflight_.store(bg_inflight - cost, std::memory_order_relaxed); - } - // Each freed page-unit can admit at most one waiter; over-waking is safe - // because woken tasks re-check the admission condition and re-wait. - // Background waiters are woken first while the sub-budget has room. - // Their demand and unused entitlement stay reserved through admission, - // including the wake-to-admit gap (see Acquire), so foreground tasks woken - // for those units simply re-check and re-wait. - size_t woken = 0; - if (bg_cap_ != 0 && bg_inflight_.load(std::memory_order_relaxed) < bg_cap_) - { - woken = bg_waiting_.WakeN(cost); - } - // Always give foreground a wake as well, not only the leftover - // credits. When background is a saturated treadmill (its queue never - // empties, so every release re-donates to background), foreground - // waiters would otherwise have no wake source once the last - // foreground in-flight completes: the class deadlocks behind - // `!zone.Empty()` admission until background's queue happens to - // drain (observed as multi-hundred-ms foreground gate stalls under - // write storms). Over-waking is safe: woken tasks re-check the - // admission condition and re-wait. - waiting_.WakeN(cost > woken ? cost - woken : 1); + WaitingZone &zone = background ? bg_waiting_ : waiting_; + if (zone.Empty() && CanAdmit(background)) + { + // Fast path: no queue and an admissible balance — charge inline. + Charge(ops, bytes, background); + return; + } + // Queue behind existing waiters of the class (FIFO within class) with + // the cost recorded for the waker: RefillAndWake peeks the FIFO head, + // charges its recorded cost and only then wakes it, so wake counts + // are exact for heterogeneous costs and a woken task never re-queues. + // GrantWaiters is the ONLY waker of these zones, so by the time this + // task resumes its cost has already been charged and there is nothing + // left to do here; a second wake source must not be added without + // extending this handshake. + KvTask *task = ThdTask(); + task->rate_wait_ops_ = ops; + task->rate_wait_bytes_ = bytes; + const uint64_t start_us = shard->ReadTimeMicroseconds(); + (background ? bg_blocked_count_ : blocked_count_) + .fetch_add(1, std::memory_order_relaxed); + zone.Wait(task); + (background ? bg_blocked_us_ : blocked_us_) + .fetch_add(shard->DurationMicroseconds(start_us), + std::memory_order_relaxed); +} + +void RateBudget::RefillAndWake(uint64_t now_us) +{ + if (now_us <= last_refill_us_) + { + return; + } + // The elapsed time is clamped to the burst window, which has three + // effects. (1) An idle gap banks at most one bucket of credit: a shard + // idle for a second does not come back with a second's worth of + // tokens, only rate x burst_us_. (2) The very first call is well + // defined: last_refill_us_ is 0, so the raw elapsed would be the + // absolute clock value; clamped, the budget simply starts with one + // full bucket of allowance. (3) rate x elapsed cannot overflow int64 + // no matter how long the gap. + const uint64_t elapsed = std::min(now_us - last_refill_us_, burst_us_); + last_refill_us_ = now_us; + auto refill = [elapsed, this](int64_t &bal, uint64_t rate) + { + if (rate == 0) + { + return; + } + const int64_t cap = static_cast(rate * burst_us_); + bal = std::min(bal + static_cast(rate * elapsed), cap); + }; + refill(fg_ops_bal_, fg_ops_rate_); + refill(fg_bytes_bal_, fg_bytes_rate_); + refill(bg_ops_bal_, bg_ops_rate_); + refill(bg_bytes_bal_, bg_bytes_rate_); + // The classes have disjoint buckets, so grant each independently; the + // admission test includes the borrow path, so a foreground waiter + // whose lender just became idle-and-positive is granted too. + GrantWaiters(false); + GrantWaiters(true); +} + +void RateBudget::GrantWaiters(bool background) +{ + // Peek-and-grant: charge the FIFO head's recorded cost on its behalf, + // mark it granted, then wake it — exact wake counts with + // heterogeneous costs, no over-waking, no re-queue churn. The debt + // rule is preserved verbatim: each charge may drive the balance + // negative, and the loop stops exactly where serialized admission + // would. + WaitingZone &zone = background ? bg_waiting_ : waiting_; + while (!zone.Empty() && CanAdmit(background)) + { + KvTask *head = zone.Head(); + Charge(head->rate_wait_ops_, head->rate_wait_bytes_, background); + zone.WakeOne(); + } } IouringMgr::IouringMgr(const KvOptions *opts, uint32_t fd_limit) @@ -340,24 +403,35 @@ IouringMgr::IouringMgr(const KvOptions *opts, uint32_t fd_limit) write_req_pool_ = std::make_unique(pool_size); merged_write_req_pool_ = std::make_unique(pool_size); - // In-flight page-IO budgets (docs/design/io_qos.md M1/M2). The request - // pools above count request objects, while the write budget counts page - // units and permits one oversized request to run alone; sharing the option - // makes pool sizing conservative but does not make the bounds identical. - // The read budget carries the background sub-budget; the write budget has - // none — all page writes come from write tasks, which are background by - // definition. - read_budget_.SetCap(options_->max_inflight_read); - write_budget_.SetCap(options_->max_inflight_write); - if (options_->max_inflight_read != 0) - { - const uint32_t ratio = - std::clamp(options_->bg_read_ratio, 1, 100); - const uint32_t bg_cap = std::max( - 1, - static_cast(uint64_t{options_->max_inflight_read} * - ratio / 100)); - read_budget_.SetBgCap(bg_cap); + // Device admission control (docs/design/io_qos.md M4): the rate budget + // (below) plus the optional class-blind in-flight command window. The + // M1/M2 count budgets are retired; max_inflight_read / bg_read_ratio + // are deprecated no-ops and max_inflight_write only sizes the request + // pools above. + io_window_cap_ = options_->max_inflight_io; + + // Device rate budget (docs/design/io_qos.md M4): the per-disk + // provisioned limits, spread across shards by simple division. Assumes + // shards spread IO uniformly across store paths (true on average via + // the store-path LUT). + if (options_->disk_rate_limit_iops != 0 || + options_->disk_rate_limit_mbps != 0) + { + const uint64_t num_disks = + std::max(1, options_->store_path.size()); + const uint64_t shards = std::max(1, options_->num_threads); + const uint64_t shard_ops = + options_->disk_rate_limit_iops * num_disks / shards; + const uint64_t shard_bytes = options_->disk_rate_limit_mbps * + num_disks * (uint64_t{1} << 20) / shards; + rate_budget_.SetRates(shard_ops, + shard_bytes, + options_->rate_limit_burst_ms, + options_->rate_bg_ratio); + LOG(INFO) << "IO rate budget: " << shard_ops << " ops/s, " + << (shard_bytes >> 20) << " MB/s per shard (" + << options_->disk_rate_limit_iops << " iops x " << num_disks + << " disks / " << shards << " shards)"; } } @@ -928,7 +1002,14 @@ std::pair IouringMgr::ReadPage(const TableIdent &tbl_id, // bounded by the BG sub-budget. const uint64_t t_gate = io_stats_enabled_ ? shard->ReadTimeMicroseconds() : 0; - read_budget_.Acquire(1, ThdTask()->IsBackground()); + // Rate-budget admission (M4) before the count budget: the rate + // bucket paces device ops/bytes per second; the count budget + // stays as the burst-depth guard closest to SQE prep. Both are + // coroutine waits with independent wake sources (time vs CQE), + // so ordering cannot deadlock. + rate_budget_.Acquire( + 1, options_->data_page_size, ThdTask()->IsBackground()); + AcquireIoWindow(1); const uint64_t t_sqe = io_stats_enabled_ ? shard->ReadTimeMicroseconds() : 0; io_uring_sqe *sqe = GetSQE(UserDataType::KvTaskPageRead, ThdTask()); @@ -1073,7 +1154,10 @@ KvError IouringMgr::ReadPages(const TableIdent &tbl_id, // Read-budget admission (io_qos.md M1/M2) is per page, not per batch: // the task may block mid-batch while already-submitted pages // complete, so a batch larger than the (sub-)budget cannot deadlock. - read_budget_.Acquire(1, req->task_->IsBackground()); + // Rate budget (M4) first, same per-page granularity. + rate_budget_.Acquire( + 1, options_->data_page_size, req->task_->IsBackground()); + AcquireIoWindow(1); auto [fd, registered] = req->fd_ref_.FdPair(); io_uring_sqe *sqe = GetSQE(UserDataType::BaseReqPageRead, req); if (registered) @@ -1220,8 +1304,12 @@ KvError IouringMgr::WritePage(const TableIdent &tbl_id, auto [fd, registered] = fd_ref.FdPair(); WriteReq *req = write_req_pool_->Alloc(std::move(fd_ref), std::move(page)); // Write-budget admission (io_qos.md M1): after every other blocking - // acquisition (FD, req pool), immediately before SQE prep. - write_budget_.Acquire(1); + // acquisition (FD, req pool), immediately before SQE prep. The rate + // budget (M4) is charged first; write tasks classify as background, + // so page writes draw from the background sub-bucket. + rate_budget_.Acquire( + 1, options_->data_page_size, ThdTask()->IsBackground()); + AcquireIoWindow(1); io_uring_sqe *sqe = GetSQE(UserDataType::WriteReq, req); if (registered) { @@ -1527,10 +1615,19 @@ KvError IouringMgr::SubmitMergedWrite(const TableIdent &tbl_id, static_cast(req->pages_.size() - 1); } - // Write-budget admission (io_qos.md M1): cost in configured data-page - // units so the cap means the same thing in append and non-append mode. - // Must mirror the release cost computed from bytes_ in PollComplete. - write_budget_.Acquire(MergedWriteCost(req->bytes_)); + // Write-budget admission (io_qos.md M1): cost in 4KB-page units so the + // cap means the same thing in append and non-append mode. Must mirror + // the release cost computed from bytes_ in PollComplete. + // Rate budget (M4) first: ops cost mirrors the kernel's split of large + // IOs into device commands of at most rate_limit_io_unit bytes; the + // bytes bucket is charged the full length. Debt admission means this + // single large acquisition never deadlocks against the bucket size. + const uint32_t io_unit = std::max(options_->rate_limit_io_unit, + options_->data_page_size); + rate_budget_.Acquire(static_cast((bytes + io_unit - 1) / io_unit), + bytes, + ThdTask()->IsBackground()); + AcquireIoWindow(DeviceCmdCost(bytes)); io_uring_sqe *sqe = GetSQE(UserDataType::MergedWriteReq, req); auto [fd, registered] = req->fd_ref_.FdPair(); if (registered) @@ -2183,6 +2280,14 @@ std::pair IouringMgr::ConvFileSegmentId( void IouringMgr::Submit() { + // Refill the device rate budget once per loop iteration (M4). Must run + // on every iteration — including no-op ones — because refill is the + // only wake source for rate-budget waiters. + if (rate_budget_.Enabled()) + { + rate_budget_.RefillAndWake(Shard::ReadTimeMicroseconds()); + } + const uint32_t prepared_before = prepared_sqe_; const uint32_t sq_flags = ring_.sq.kflags == nullptr ? 0 : *ring_.sq.kflags; const bool need_taskrun = (sq_flags & IORING_SQ_TASKRUN) != 0; @@ -2272,7 +2377,7 @@ void IouringMgr::PollComplete() if (type == UserDataType::KvTaskPageRead) { TEST_FAIL_POINT_ACTION("KvTaskPageReadCqe", cqe->res = -EIO); - read_budget_.Release(1, task->IsBackground()); + ReleaseIoWindow(1); if (io_stats_enabled_) { task->op_cqe_us_ = loop_now_us_; @@ -2288,7 +2393,7 @@ void IouringMgr::PollComplete() if (type == UserDataType::BaseReqPageRead) { TEST_FAIL_POINT_ACTION("BaseReqPageReadCqe", cqe->res = -EIO); - read_budget_.Release(1, req->task_->IsBackground()); + ReleaseIoWindow(1); } req->res_ = cqe->res; req->flags_ = cqe->flags; @@ -2316,12 +2421,7 @@ void IouringMgr::PollComplete() req->task_->WritePageCallback(std::move(req->page_), err); task = req->task_; write_req_pool_->Free(req); - // No class argument: the write budget has no background - // sub-budget (all page writes come from write tasks, i.e. - // background — see io_qos.md M2), so acquire and release both - // use the default. Must stay symmetric with WritePage's - // Acquire. - write_budget_.Release(1); + ReleaseIoWindow(1); break; } case UserDataType::MergedWriteReq: @@ -2356,10 +2456,8 @@ void IouringMgr::PollComplete() req->release_indices_[i]); } } - // No class argument (see the WriteReq case): the write budget - // has no background sub-budget. Cost must mirror - // SubmitMergedWrite's Acquire exactly. - write_budget_.Release(MergedWriteCost(req->bytes_)); + // Cost must mirror SubmitMergedWrite's AcquireIoWindow exactly. + ReleaseIoWindow(DeviceCmdCost(req->bytes_)); merged_write_req_pool_->Free(req); continue; } @@ -2620,6 +2718,10 @@ KvError IouringMgr::FdatasyncFiles(const TableIdent &tbl_id, // Fsync all dirty files/directory. std::vector reqs; reqs.reserve(fds.size()); + // Rate budget (M4): one device op per fsync, no bytes. Other metadata + // ops (open, statx, rename, ...) stay exempt, matching M1. + rate_budget_.Acquire( + static_cast(fds.size()), 0, ThdTask()->IsBackground()); // Instrumented for IO QoS evaluation (io_qos.md): fsync stalls are a // distinct interference channel from page-IO queueing. const uint64_t fsync_start_us = shard->ReadTimeMicroseconds(); diff --git a/src/kv_options.cpp b/src/kv_options.cpp index 08f43f5d4..200661d2a 100644 --- a/src/kv_options.cpp +++ b/src/kv_options.cpp @@ -156,12 +156,45 @@ int KvOptions::LoadFromIni(const char *path) } if (reader.HasValue(sec_run, "max_inflight_read")) { - max_inflight_read = - reader.GetUnsigned(sec_run, "max_inflight_read", 64); + max_inflight_read = reader.GetUnsigned(sec_run, "max_inflight_read", 0); + LOG(WARNING) << "max_inflight_read is deprecated and has no effect; " + "device admission control is disk_rate_limit_iops " + "(with rate_bg_ratio) and max_inflight_io"; } if (reader.HasValue(sec_run, "bg_read_ratio")) { bg_read_ratio = reader.GetUnsigned(sec_run, "bg_read_ratio", 25); + LOG(WARNING) << "bg_read_ratio is deprecated and has no effect; " + "the background share of the device rate budget is " + "rate_bg_ratio"; + } + if (reader.HasValue(sec_run, "disk_rate_limit_iops")) + { + disk_rate_limit_iops = + reader.GetUnsigned64(sec_run, "disk_rate_limit_iops", 275'000); + } + if (reader.HasValue(sec_run, "disk_rate_limit_mbps")) + { + disk_rate_limit_mbps = + reader.GetUnsigned64(sec_run, "disk_rate_limit_mbps", 0); + } + if (reader.HasValue(sec_run, "rate_limit_burst_ms")) + { + rate_limit_burst_ms = + reader.GetUnsigned(sec_run, "rate_limit_burst_ms", 4); + } + if (reader.HasValue(sec_run, "rate_limit_io_unit")) + { + std::string io_unit_str = reader.Get(sec_run, "rate_limit_io_unit", ""); + rate_limit_io_unit = ParseSizeWithUnit(io_unit_str); + } + if (reader.HasValue(sec_run, "rate_bg_ratio")) + { + rate_bg_ratio = reader.GetUnsigned(sec_run, "rate_bg_ratio", 25); + } + if (reader.HasValue(sec_run, "max_inflight_io")) + { + max_inflight_io = reader.GetUnsigned(sec_run, "max_inflight_io", 0); } if (reader.HasValue(sec_run, "max_write_batch_pages")) { @@ -169,8 +202,8 @@ int KvOptions::LoadFromIni(const char *path) reader.GetUnsigned(sec_run, "max_write_batch_pages", 64); LOG(WARNING) << "Option max_write_batch_pages is deprecated and has no " - "effect; in-flight write IO is bounded by max_inflight_write " - "(see docs/design/io_qos.md)"; + "effect; device write admission is paced by the rate budget " + "(disk_rate_limit_iops, see docs/design/io_qos.md)"; } if (reader.HasValue(sec_run, "coroutine_stack_size")) { @@ -413,6 +446,12 @@ bool KvOptions::operator==(const KvOptions &other) const max_inflight_write == other.max_inflight_write && max_inflight_read == other.max_inflight_read && bg_read_ratio == other.bg_read_ratio && + disk_rate_limit_iops == other.disk_rate_limit_iops && + disk_rate_limit_mbps == other.disk_rate_limit_mbps && + rate_limit_burst_ms == other.rate_limit_burst_ms && + rate_limit_io_unit == other.rate_limit_io_unit && + rate_bg_ratio == other.rate_bg_ratio && + max_inflight_io == other.max_inflight_io && max_write_batch_pages == other.max_write_batch_pages && coroutine_stack_size == other.coroutine_stack_size && num_retained_archives == other.num_retained_archives && diff --git a/src/storage/shard.cpp b/src/storage/shard.cpp index ae8b290c5..1faa3ee04 100644 --- a/src/storage/shard.cpp +++ b/src/storage/shard.cpp @@ -139,13 +139,14 @@ void Shard::CollectPeriodicGauges(metrics::Meter *meter) meter->Collect(metrics::NAME_ELOQSTORE_LOCAL_SPACE_USED, static_cast(io_mgr_->GetLocalSpaceUsed())); + // The M1/M2 count budgets (and their per-class in-flight gauges) are + // retired in favor of the M4 rate budget (docs/design/io_qos.md). The + // closest surviving instantaneous-depth gauge is the class-blind + // in-flight device-command window; report it under the read-pages + // metric name until dedicated rate-budget metrics are defined. const IoQosStats qos = io_mgr_->GetIoQosStats(); meter->Collect(metrics::NAME_ELOQSTORE_INFLIGHT_READ_PAGES, - static_cast(qos.read_.inflight_)); - meter->Collect(metrics::NAME_ELOQSTORE_INFLIGHT_BG_READ_PAGES, - static_cast(qos.bg_read_.inflight_)); - meter->Collect(metrics::NAME_ELOQSTORE_INFLIGHT_WRITE_PAGES, - static_cast(qos.write_.inflight_)); + static_cast(qos.io_window_inflight_)); } #endif diff --git a/tests/io_qos.cpp b/tests/io_qos.cpp index 6312e4a40..0152cb564 100644 --- a/tests/io_qos.cpp +++ b/tests/io_qos.cpp @@ -1,16 +1,20 @@ /** - * IO QoS (docs/design/io_qos.md) — M1 in-flight page-IO budget tests. + * IO QoS (docs/design/io_qos.md) — M4 device rate budget and in-flight + * command window tests. * - * These tests run with deliberately tiny caps so the blocking paths are hot, - * then assert the accounting invariants: budgets drain to zero at quiesce, - * high-watermarks respect the caps (except the documented oversized-request - * admission), and disabled budgets stay untouched. + * Rates and windows are set deliberately tiny so the blocking paths are + * hot, then the tests assert: pacing completes without deadlock (debt + * admission, refill-driven wakes), classes are charged correctly, + * borrowing is foreground-only, disabled mechanisms stay zero-cost, and + * shutdown drains queued waiters cleanly. */ #include +#include #include #include #include +#include #include "async_io_manager.h" #include "common.h" @@ -32,343 +36,149 @@ eloqstore::IoQosStats ShardStats(const eloqstore::EloqStore *store) } } // namespace -TEST_CASE("io budgets: accounting invariants under tiny caps", "[io_qos]") +TEST_CASE("defaults: rate limiting on, unit workload never blocks", "[io_qos]") { - eloqstore::KvOptions opts = default_opts; - opts.max_inflight_read = 4; - opts.max_inflight_write = 8; - eloqstore::EloqStore *store = InitStore(opts); + // Rate limiting is on by default (275K IOPS per disk / shard here): + // it must charge a trivial single-threaded workload without ever + // blocking it, and the io window is off by default. + eloqstore::EloqStore *store = InitStore(default_opts); MapVerifier verify(test_tbl_id, store, false); verify.SetValueSize(200); - for (int round = 0; round < 4; round++) + verify.WriteRnd(0, 2000, 0, 25); + for (int i = 0; i < 100; i++) { - verify.WriteRnd(0, 2000, 0, 25); - for (int i = 0; i < 100; i++) - { - verify.Read(std::rand() % 2000); - } - verify.Scan(0, 300); + verify.Read(std::rand() % 2000); } eloqstore::IoQosStats stats = ShardStats(store); - // Budgets drain to zero once all requests have completed. - REQUIRE(stats.read_.inflight_ == 0); - REQUIRE(stats.write_.inflight_ == 0); - // All page IO in this mode has cost 1, so watermarks are hard-capped. - REQUIRE(stats.read_.high_watermark_ >= 1); - REQUIRE(stats.read_.high_watermark_ <= 4); - REQUIRE(stats.write_.high_watermark_ >= 1); - REQUIRE(stats.write_.high_watermark_ <= 8); - REQUIRE(stats.read_.admitted_pages_ > 0); - REQUIRE(stats.write_.admitted_pages_ > 0); - REQUIRE(stats.fdatasync_count_ > 0); - REQUIRE(stats.fdatasync_us_ > 0); + REQUIRE(stats.rate_.admitted_ops_ > 0); + REQUIRE(stats.rate_.blocked_count_ == 0); + REQUIRE(stats.io_window_blocked_ == 0); + REQUIRE(stats.io_window_hwm_ == 0); } -TEST_CASE("io budgets: overflow read batch larger than the cap", "[io_qos]") +TEST_CASE("rate budget: iops = 0 disables it, stats stay zero", "[io_qos]") { - // 600KB values span ~150 overflow pages; with overflow_pointers = 128, - // GetOverflowValue issues 128-page ReadPages batches — far above the - // 4-page read cap. Per-page acquisition must make progress regardless. + // Rate limiting is ON by default (275K IOPS per disk, a cloud-NVMe + // starting point); zero must disable it entirely. + REQUIRE(eloqstore::KvOptions{}.disk_rate_limit_iops == 275'000); eloqstore::KvOptions opts = default_opts; - opts.max_inflight_read = 4; - opts.max_inflight_write = 8; - opts.overflow_pointers = 128; - eloqstore::EloqStore *store = InitStore(opts); - - MapVerifier verify(test_tbl_id, store, false); - verify.SetValueSize(600 * 1024); - verify.Upsert(1); - verify.Upsert(2); - verify.Read(1); - verify.Read(2); - - eloqstore::IoQosStats stats = ShardStats(store); - REQUIRE(stats.read_.inflight_ == 0); - REQUIRE(stats.write_.inflight_ == 0); - // Cost-1 reads: the cap is strict even for oversized batches. - REQUIRE(stats.read_.high_watermark_ <= 4); - // A 128-page batch through a 4-page budget must have waited. - REQUIRE(stats.read_.blocked_count_ > 0); - REQUIRE(stats.read_.blocked_us_ > 0); -} - -TEST_CASE("io budgets: merged append writes and oversized admission", - "[io_qos]") -{ - // Append mode aggregates page writes into ~1MB merged writes - // (cost = 256 pages at 4KB). With a 64-page write cap, each merged - // write exceeds the cap and is admitted alone once the budget drains: - // in-flight is bounded by the single-request cost, not the cap. - eloqstore::KvOptions opts = append_opts; - opts.max_inflight_read = 4; - opts.max_inflight_write = 64; + opts.disk_rate_limit_iops = 0; eloqstore::EloqStore *store = InitStore(opts); MapVerifier verify(test_tbl_id, store, false); - verify.SetValueSize(4000); - verify.WriteRnd(0, 3000, 0, 50); + verify.SetValueSize(200); + verify.WriteRnd(0, 1000, 0, 25); for (int i = 0; i < 50; i++) { - verify.Read(std::rand() % 3000); - } - - const uint32_t merged_cost_bound = - opts.write_buffer_size / opts.data_page_size; - eloqstore::IoQosStats stats = ShardStats(store); - REQUIRE(stats.read_.inflight_ == 0); - REQUIRE(stats.write_.inflight_ == 0); - REQUIRE(stats.write_.high_watermark_ >= 1); - REQUIRE(stats.write_.high_watermark_ <= merged_cost_bound); -} - -TEST_CASE("io budgets: cap 512 admits two concurrent merged writes", "[io_qos]") -{ - // Counterpart of the oversized-admission test: with the cap at twice - // the merged-write cost (512 vs 256 pages), merged writes from - // concurrent write tasks may overlap in flight — the watermark must - // exceed one merged write's cost — while never exceeding the cap. - // (A single task's flushes do not reliably overlap: it yields per page - // while building the next buffer, so concurrency comes from multiple - // partitions' write tasks on one shard.) - eloqstore::KvOptions opts = append_opts; - opts.num_threads = 1; - opts.max_inflight_write = 512; - eloqstore::EloqStore *store = InitStore(opts); - - constexpr uint32_t num_parts = 8; - constexpr uint32_t keys_per_part = 1200; // ~1200 pages = ~5 flushes - const uint64_t ts = utils::UnixTs(); - std::array reqs; - std::atomic done{0}; - for (uint32_t p = 0; p < num_parts; p++) - { - std::vector entries; - entries.reserve(keys_per_part); - for (uint32_t i = 0; i < keys_per_part; i++) - { - entries.emplace_back(test_util::Key(i, 7), - std::string(3000, 'w'), - ts, - eloqstore::WriteOp::Upsert); - } - reqs[p].SetArgs(eloqstore::TableIdent("qos-dual", p), - std::move(entries)); - store->ExecAsyn(&reqs[p], - 0, - [&done](eloqstore::KvRequest *) - { done.fetch_add(1, std::memory_order_relaxed); }); - } - while (done.load(std::memory_order_relaxed) < int(num_parts)) - { - std::this_thread::sleep_for(std::chrono::milliseconds(5)); - } - for (auto &req : reqs) - { - REQUIRE(req.Error() == eloqstore::KvError::NoError); - } - - const uint32_t merged_cost = - opts.write_buffer_size / opts.data_page_size; // 256 - eloqstore::IoQosStats stats = ShardStats(store); - REQUIRE(stats.write_.inflight_ == 0); - REQUIRE(stats.write_.high_watermark_ > merged_cost); - REQUIRE(stats.write_.high_watermark_ <= 512); -} - -TEST_CASE("io budgets: failed write drains the budget", "[io_qos]") -{ - // Mid-batch write failure: make the partition directory read-only after - // the first data file exists, then write a batch large enough to need a - // second file. Creating that file fails (EACCES) with merged writes to - // the first file still in flight; the task aborts, and AbortWrite's - // WaitIo must drain every in-flight page so the budgets return to zero - // and the store stays usable. - namespace fs = std::filesystem; - eloqstore::KvOptions opts = append_opts; // 1MB files (2^8 pages) - eloqstore::EloqStore *store = InitStore(opts); - - eloqstore::TableIdent tbl_id{"qos-fail", 0}; - MapVerifier verify(tbl_id, store, false); - verify.SetValueSize(3000); // one KV per page - verify.Upsert(0, 10); // creates the partition dir + file 0 - - const fs::path part_dir = fs::path(test_path) / tbl_id.ToString(); - REQUIRE(fs::exists(part_dir)); - fs::permissions(part_dir, - fs::perms::owner_read | fs::perms::owner_exec, - fs::perm_options::replace); - - // ~600 pages: fills file 0 (256 pages) and needs file 1 -> EACCES. - std::vector entries; - const uint64_t ts = utils::UnixTs(); - for (uint32_t i = 100; i < 700; i++) - { - entries.emplace_back(std::to_string(1000000 + i), - std::string(3000, 'x'), - ts, - eloqstore::WriteOp::Upsert); + verify.Read(std::rand() % 1000); } - eloqstore::BatchWriteRequest fail_req; - fail_req.SetArgs(tbl_id, std::move(entries)); - store->ExecSync(&fail_req); - fs::permissions(part_dir, fs::perms::owner_all, fs::perm_options::replace); - REQUIRE(fail_req.Error() != eloqstore::KvError::NoError); eloqstore::IoQosStats stats = ShardStats(store); - REQUIRE(stats.write_.inflight_ == 0); - REQUIRE(stats.read_.inflight_ == 0); - REQUIRE(stats.bg_read_.inflight_ == 0); - - // The store must remain usable after the abort. - eloqstore::TableIdent recover_tbl{"qos-fail", 1}; - MapVerifier recover(recover_tbl, store, false); - recover.SetValueSize(200); - recover.Upsert(0, 50); - recover.Read(7); + REQUIRE(stats.rate_.admitted_ops_ == 0); + REQUIRE(stats.rate_.blocked_count_ == 0); + REQUIRE(stats.bg_rate_.blocked_count_ == 0); } -TEST_CASE("io budgets: negative WriteReq CQE drains and recovers", "[io_qos]") +TEST_CASE("rate budget: charges device IO and completes under a tiny rate", + "[io_qos]") { + // A deliberately low rate (well below what the workload wants) must + // pace the run without deadlock or error: debt admission guarantees + // progress for costs larger than the bucket, and refill-driven wakes + // guarantee waiter progress. Blocked counters must move, every op + // must be accounted, and balances must drain (no in-flight concept — + // admitted ops are cumulative). eloqstore::KvOptions opts = default_opts; - opts.max_inflight_write = 8; + opts.disk_rate_limit_iops = 2000; // per "disk"; one path, one shard + opts.rate_limit_burst_ms = 4; eloqstore::EloqStore *store = InitStore(opts); - const eloqstore::TableIdent tbl_id{"qos-write-cqe", 0}; - const uint64_t ts = utils::UnixTs(); - eloqstore::BatchWriteRequest failed; - failed.SetTableId(tbl_id); - for (uint32_t i = 0; i < 100; ++i) + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(200); + verify.WriteRnd(0, 500, 0, 25); + for (int i = 0; i < 50; i++) { - failed.AddWrite(test_util::Key(i, 7), - std::string(200, 'w'), - ts, - eloqstore::WriteOp::Upsert); + verify.Read(std::rand() % 500); } - eloqstore::FailPoint::GetInstance().ArmOnce("WriteReqCqe"); - store->ExecSync(&failed); - eloqstore::FailPoint::GetInstance().Disarm(); - REQUIRE(failed.Error() == eloqstore::KvError::IoFail); - REQUIRE(ShardStats(store).write_.inflight_ == 0); - - eloqstore::BatchWriteRequest recovery; - recovery.SetTableId(tbl_id); - recovery.AddWrite("recovery", "value", ts + 1, eloqstore::WriteOp::Upsert); - store->ExecSync(&recovery); - REQUIRE(recovery.Error() == eloqstore::KvError::NoError); - REQUIRE(ShardStats(store).write_.inflight_ == 0); + eloqstore::IoQosStats stats = ShardStats(store); + // The budget is partitioned per class: rate_ meters foreground reads, + // bg_rate_ meters background IO (background reads and all write-path + // IO). Both classes did device IO here, so both must show charges. + REQUIRE(stats.rate_.admitted_ops_ > 0); + REQUIRE(stats.rate_.admitted_bytes_ > 0); + REQUIRE(stats.bg_rate_.admitted_ops_ > 0); + REQUIRE(stats.bg_rate_.admitted_bytes_ > 0); } -TEST_CASE("io budgets: negative MergedWriteReq CQE drains and recovers", +TEST_CASE("rate budget: foreground borrows background's idle surplus", "[io_qos]") { - eloqstore::KvOptions opts = append_opts; - opts.max_inflight_write = 64; + // Reads run after all writes have completed, so the background class + // is idle and its share should be lent to foreground: with a rate low + // enough that foreground exhausts its own 75% share, some read + // admissions must be granted from the background bucket (borrowed), + // and every op must still be accounted to the borrower's class. + eloqstore::KvOptions opts = default_opts; + opts.disk_rate_limit_iops = 2000; + opts.rate_limit_burst_ms = 4; eloqstore::EloqStore *store = InitStore(opts); - const eloqstore::TableIdent tbl_id{"qos-merged-cqe", 0}; - const uint64_t ts = utils::UnixTs(); - eloqstore::BatchWriteRequest failed; - failed.SetTableId(tbl_id); - for (uint32_t i = 0; i < 400; ++i) + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(200); + verify.WriteRnd(0, 500, 0, 25); + for (int i = 0; i < 100; i++) { - failed.AddWrite(test_util::Key(i, 7), - std::string(3000, 'm'), - ts, - eloqstore::WriteOp::Upsert); + verify.Read(std::rand() % 500); } - eloqstore::FailPoint::GetInstance().ArmOnce("MergedWriteReqCqe"); - store->ExecSync(&failed); - eloqstore::FailPoint::GetInstance().Disarm(); - - REQUIRE(failed.Error() == eloqstore::KvError::IoFail); - REQUIRE(ShardStats(store).write_.inflight_ == 0); - - eloqstore::BatchWriteRequest recovery; - recovery.SetTableId(tbl_id); - recovery.AddWrite("recovery", "value", ts + 1, eloqstore::WriteOp::Upsert); - store->ExecSync(&recovery); - REQUIRE(recovery.Error() == eloqstore::KvError::NoError); - REQUIRE(ShardStats(store).write_.inflight_ == 0); -} -TEST_CASE("io budgets: negative KvTaskPageRead CQE drains and recovers", - "[io_qos]") -{ - eloqstore::KvOptions opts = default_opts; - opts.max_inflight_read = 4; - eloqstore::EloqStore *store = InitStore(opts); - const eloqstore::TableIdent tbl_id{"qos-read-cqe", 0}; - MapVerifier seed(tbl_id, store, false); - seed.SetValueSize(200); - seed.Upsert(0, 100); - const std::string key = test_util::Key(7, 7); - const std::string expected = seed.DataSet().at(key).value_; - - eloqstore::ReadRequest failed; - failed.SetArgs(tbl_id, key); - eloqstore::FailPoint::GetInstance().ArmOnce("KvTaskPageReadCqe"); - store->ExecSync(&failed); - eloqstore::FailPoint::GetInstance().Disarm(); - - REQUIRE(failed.Error() == eloqstore::KvError::IoFail); - REQUIRE(ShardStats(store).read_.inflight_ == 0); - - eloqstore::ReadRequest recovery; - recovery.SetArgs(tbl_id, key); - store->ExecSync(&recovery); - REQUIRE(recovery.Error() == eloqstore::KvError::NoError); - REQUIRE(recovery.value_ == expected); - REQUIRE(ShardStats(store).read_.inflight_ == 0); + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.rate_.admitted_ops_ > 0); + REQUIRE(stats.rate_.borrowed_ops_ > 0); + // Borrowed ops are a subset of the class's admitted ops. + REQUIRE(stats.rate_.borrowed_ops_ <= stats.rate_.admitted_ops_); + // Borrowing is asymmetric: background must never borrow foreground's + // share (symmetric borrowing collapsed storm isolation — see + // RateBudget::CanAdmit). + REQUIRE(stats.bg_rate_.borrowed_ops_ == 0); } -TEST_CASE("io budgets: negative BaseReqPageRead CQE drains and recovers", +TEST_CASE("rate budget: overflow read batch far above one burst of tokens", "[io_qos]") { + // 600KB values span ~150 overflow pages; GetOverflowValue issues + // 128-page ReadPages batches. At 2000 units/s with a 4 ms bucket + // (8 banked tokens) a batch is far above one burst, so per-page + // acquisition must pace it through debt and refills without deadlock. eloqstore::KvOptions opts = default_opts; - opts.max_inflight_read = 4; + opts.disk_rate_limit_iops = 2000; + opts.rate_limit_burst_ms = 4; opts.overflow_pointers = 128; eloqstore::EloqStore *store = InitStore(opts); - const eloqstore::TableIdent tbl_id{"qos-batch-read-cqe", 0}; - MapVerifier seed(tbl_id, store, false); - constexpr uint32_t value_size = 32 * 1024; - seed.SetValueSize(value_size); - seed.Upsert(0); - const std::string key = test_util::Key(0, 7); - const std::string expected = seed.DataSet().at(key).value_; - eloqstore::ReadRequest failed; - failed.SetArgs(tbl_id, key); - eloqstore::FailPoint::GetInstance().ArmOnce("BaseReqPageReadCqe"); - store->ExecSync(&failed); - eloqstore::FailPoint::GetInstance().Disarm(); + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(600 * 1024); + verify.Upsert(1); + verify.Upsert(2); + verify.Read(1); + verify.Read(2); - REQUIRE(failed.Error() == eloqstore::KvError::IoFail); eloqstore::IoQosStats stats = ShardStats(store); - REQUIRE(stats.read_.inflight_ == 0); - REQUIRE(stats.bg_read_.inflight_ == 0); - - eloqstore::ReadRequest recovery; - recovery.SetArgs(tbl_id, key); - store->ExecSync(&recovery); - REQUIRE(recovery.Error() == eloqstore::KvError::NoError); - REQUIRE(recovery.value_ == expected); - stats = ShardStats(store); - REQUIRE(stats.read_.inflight_ == 0); - REQUIRE(stats.bg_read_.inflight_ == 0); + REQUIRE(stats.rate_.admitted_ops_ > 0); + REQUIRE(stats.rate_.blocked_count_ > 0); } -TEST_CASE("io budgets: shutdown while tasks queue behind the budget", - "[io_qos]") +TEST_CASE("rate budget: shutdown while tasks queue behind the gate", "[io_qos]") { - // Several overflow reads (128-page batches) contend for a 1-page read - // budget, then the store is stopped while they are still queued. Stop - // must drain cleanly (no hang, no crash) and every request must - // complete. + // Several overflow reads contend for a tiny rate, then the store is + // stopped while they are still queued at the gate. Stop must drain + // cleanly (refill-driven wakes depend only on the shard loop running) + // and every request must complete. eloqstore::KvOptions opts = default_opts; - opts.max_inflight_read = 1; // bg_cap clamps to 1 as well + opts.disk_rate_limit_iops = 2000; + opts.rate_limit_burst_ms = 4; opts.overflow_pointers = 128; eloqstore::EloqStore *store = InitStore(opts); @@ -395,360 +205,123 @@ TEST_CASE("io budgets: shutdown while tasks queue behind the budget", { REQUIRE(req.Error() == eloqstore::KvError::NoError); } - eloqstore::IoQosStats stats = ShardStats(store); - REQUIRE(stats.read_.inflight_ == 0); - REQUIRE(stats.write_.inflight_ == 0); -} - -TEST_CASE("io budgets: disabled read budget stays untouched", "[io_qos]") -{ - eloqstore::KvOptions opts = default_opts; - opts.max_inflight_read = 0; // disabled - eloqstore::EloqStore *store = InitStore(opts); - - MapVerifier verify(test_tbl_id, store, false); - verify.SetValueSize(200); - verify.WriteRnd(0, 1000, 0, 25); - for (int i = 0; i < 50; i++) - { - verify.Read(std::rand() % 1000); - } - - eloqstore::IoQosStats stats = ShardStats(store); - REQUIRE(stats.read_.inflight_ == 0); - REQUIRE(stats.read_.high_watermark_ == 0); - REQUIRE(stats.read_.blocked_count_ == 0); - // The effectively-unbounded default write budget still counts. - REQUIRE(stats.write_.inflight_ == 0); - REQUIRE(stats.write_.blocked_count_ == 0); -} - -TEST_CASE("bg sub-budget: compaction batch reads are bounded", "[io_qos]") -{ - // Append mode with repeated 3-of-5 partial overwrites drives space - // amplification past file_amplify_factor, so the shard schedules - // compaction between batch writes (per-table writes serialize behind the - // internal compact request, so by the time the last sync write returns, - // earlier compactions have completed). Compaction move batches issue up - // to 128-page ReadPages bursts from a BackgroundWrite task — the - // BaseReqPageRead BG path — which must stay within the BG sub-budget - // (25% of 8 = 2 pages). Separate tests cover foreground capacity and - // concurrent foreground/background admission. - eloqstore::KvOptions opts = append_opts; - opts.file_amplify_factor = 2; - opts.max_inflight_read = 8; - opts.bg_read_ratio = 25; // bg_cap = 2 - eloqstore::EloqStore *store = InitStore(opts); - - MapVerifier verify(test_tbl_id, store, false); - // ~3000B values → one KV per 4KB data page, so key granularity equals - // page granularity and the overwrite pattern below controls per-file - // liveness exactly. (A fully-overwritten file is simply dropped by - // compaction with no page moves — the strided pattern keeps every file - // 40% live, i.e. SAF 2.5 > file_amplify_factor, forcing real moves.) - verify.SetValueSize(3000); - constexpr uint64_t num_keys = 1000; - verify.Upsert(0, num_keys); - for (int round = 0; round < 2; round++) - { - // Overwrite 3 of every 5 pages, uniformly across all files. - for (uint64_t base = 0; base < num_keys; base += 5) - { - verify.Upsert(base, base + 3); - } - } - for (int i = 0; i < 50; i++) - { - verify.Read(std::rand() % num_keys); - } - - eloqstore::IoQosStats stats = ShardStats(store); - REQUIRE(stats.read_.inflight_ == 0); - REQUIRE(stats.bg_read_.inflight_ == 0); - REQUIRE(stats.write_.inflight_ == 0); - // Compaction ran and its reads were charged to the BG class... - REQUIRE(stats.bg_read_.high_watermark_ >= 1); - REQUIRE(stats.bg_read_.admitted_pages_ > 0); - // ...and never exceeded the sub-budget. - REQUIRE(stats.bg_read_.high_watermark_ <= 2); - // A 128-page move batch through a 2-page sub-budget must have waited. - REQUIRE(stats.bg_read_.blocked_count_ > 0); - // Total budget still respected. - REQUIRE(stats.read_.high_watermark_ <= 8); } -TEST_CASE("bg sub-budget: foreground reads use the full budget", "[io_qos]") +TEST_CASE("io window: class-blind device-command cap bounds and completes", + "[io_qos]") { - // Foreground overflow reads (128-page batches) may exceed the BG cap and - // climb to the full read budget; only background is confined. + // A tiny window must bound in-flight device commands while everything + // still completes. 600KB values make GetOverflowValue issue 128-page + // ReadPages batches — far above a 2-command window — so per-command + // acquisition must block and make progress. No rate budget: the + // window must work standalone. (Point writes/reads alone cannot + // contend a per-shard window: one partition serializes its write + // task and the verifier reads synchronously.) eloqstore::KvOptions opts = default_opts; - opts.max_inflight_read = 8; - opts.bg_read_ratio = 25; // bg_cap = 2 + opts.disk_rate_limit_iops = 0; // window standalone, rate off + opts.max_inflight_io = 2; opts.overflow_pointers = 128; eloqstore::EloqStore *store = InitStore(opts); MapVerifier verify(test_tbl_id, store, false); verify.SetValueSize(600 * 1024); - verify.Upsert(1); // fresh key: no tree reads, so no BG read traffic - verify.Read(1); // FG: 128-page overflow batches through cap 8 - - eloqstore::IoQosStats stats = ShardStats(store); - REQUIRE(stats.read_.inflight_ == 0); - // FG climbed past the BG cap — proof the sub-budget does not bind FG. - REQUIRE(stats.read_.high_watermark_ > 2); - REQUIRE(stats.read_.high_watermark_ <= 8); - // Nothing was charged to the BG class. - REQUIRE(stats.bg_read_.high_watermark_ == 0); - REQUIRE(stats.bg_read_.blocked_count_ == 0); -} - -TEST_CASE("bg sub-budget: batch-write leaf loads are background", "[io_qos]") -{ - // Overwriting existing keys forces the BatchWrite task to load leaf data - // pages from disk (single-page KvTaskPageRead path). BatchWrite is - // classified background, so those loads are charged to the sub-budget. - eloqstore::KvOptions opts = default_opts; - opts.max_inflight_read = 8; - opts.bg_read_ratio = 25; // bg_cap = 2 - eloqstore::EloqStore *store = InitStore(opts); - - MapVerifier verify(test_tbl_id, store, false); - verify.SetValueSize(200); - verify.WriteRnd(0, 2000, 0, 100); // initial load - verify.WriteRnd(0, 2000, 0, 100); // overwrite: leaf loads from disk + verify.Upsert(1); + verify.Upsert(2); + verify.Read(1); + verify.Read(2); eloqstore::IoQosStats stats = ShardStats(store); - REQUIRE(stats.read_.inflight_ == 0); - REQUIRE(stats.bg_read_.inflight_ == 0); - REQUIRE(stats.bg_read_.high_watermark_ >= 1); - REQUIRE(stats.bg_read_.high_watermark_ <= 2); + // The window drains to zero once all requests have completed. + REQUIRE(stats.io_window_inflight_ == 0); + // A 128-page batch through a 2-command window must have waited. + REQUIRE(stats.io_window_blocked_ > 0); + // Page IO respects the cap strictly (cost-1 commands); only a merged + // write's oversized admission (ceil(1MB/256KB) = 4 > 2, admitted + // alone) may exceed it. + REQUIRE(stats.io_window_hwm_ >= 2); + REQUIRE(stats.io_window_hwm_ <= 4); } -TEST_CASE("bg sub-budget: full reservation survives the wake gap", "[io_qos]") +TEST_CASE("io window: negative KvTaskPageRead CQE releases and recovers", + "[io_qos]") { - // Keep the scheduler in the high-priority loop after the fail point moves - // the woken BG task to low priority. This makes the intended ordering - // independent of the process-wide round-budget flag. - google::FlagSaver scheduler_flag_saver; - eloqstore::FLAGS_max_processing_time_microseconds = - std::numeric_limits::max(); - + // Error-path release symmetry: a failed read CQE must still release + // its window command, the failure must surface as IoFail, and the + // store must keep serving afterwards. (Ported from the count-budget + // negative-CQE tests; the rate budget has no completion-time release + // by design, so the window carries this invariant now.) eloqstore::KvOptions opts = default_opts; - opts.num_threads = 1; - opts.max_inflight_read = 1; - opts.bg_read_ratio = 100; // pending BG demand reserves the full cap - opts.overflow_pointers = 128; + opts.disk_rate_limit_iops = 0; + opts.max_inflight_io = 4; eloqstore::EloqStore *store = InitStore(opts); + const eloqstore::TableIdent tbl_id{"qos-read-cqe", 0}; + MapVerifier seed(tbl_id, store, false); + seed.SetValueSize(200); + seed.Upsert(0, 100); + const std::string key = test_util::Key(7, 7); + const std::string expected = seed.DataSet().at(key).value_; - const eloqstore::TableIdent fg_tbl{"qos-wake-fg", 0}; - MapVerifier fg_seed(fg_tbl, store, false); - fg_seed.SetValueSize(600 * 1024); - fg_seed.Upsert(0); - - const eloqstore::TableIdent bg_tbl{"qos-wake-bg", 0}; - MapVerifier bg_seed(bg_tbl, store, false); - bg_seed.SetValueSize(200); - bg_seed.Upsert(0, 2000); - - std::atomic stop_fg{false}; - std::atomic fg_failed{false}; - std::atomic fg_started{0}; - - std::vector readers; - readers.reserve(8); - for (int i = 0; i < 8; ++i) - { - readers.emplace_back( - [&] - { - fg_started.fetch_add(1, std::memory_order_relaxed); - while (!stop_fg.load(std::memory_order_relaxed)) - { - eloqstore::ReadRequest req; - req.SetArgs(fg_tbl, test_util::Key(0, 7)); - store->ExecSync(&req); - if (req.Error() != eloqstore::KvError::NoError) - { - fg_failed.store(true, std::memory_order_relaxed); - } - } - }); - } - - auto wait_until = [](auto &&condition, std::chrono::milliseconds timeout) - { - const auto deadline = std::chrono::steady_clock::now() + timeout; - while (!condition()) - { - if (std::chrono::steady_clock::now() >= deadline) - { - return false; - } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - return true; - }; - - eloqstore::IoQosStats primed_stats; - const bool fg_primed = wait_until( - [&] - { - primed_stats = ShardStats(store); - return fg_started.load(std::memory_order_relaxed) == - readers.size() && - primed_stats.read_.inflight_ == opts.max_inflight_read && - primed_stats.read_.blocked_count_ >= - readers.size() - opts.max_inflight_read; - }, - std::chrono::seconds(2)); - - eloqstore::BatchWriteRequest bg_req; - bg_req.SetTableId(bg_tbl); - bg_req.AddWrite(test_util::Key(1000, 7), - std::string(200, 'b'), - utils::UnixTs() + 1, - eloqstore::WriteOp::Upsert); - std::atomic bg_done{false}; - bool bg_issued = false; - if (fg_primed) - { - eloqstore::FailPoint::GetInstance().ArmPersistentPaused( - "IoBudgetBgWake"); - bg_issued = store->ExecAsyn( - &bg_req, - 0, - [&bg_done](eloqstore::KvRequest *) - { bg_done.store(true, std::memory_order_relaxed); }); - } - const bool wake_gap_observed = - bg_issued && - wait_until( - [&] { return eloqstore::FailPoint::GetInstance().PauseReached(); }, - std::chrono::seconds(2)); - const bool bg_incomplete_at_barrier = - !bg_done.load(std::memory_order_relaxed); - const eloqstore::IoQosStats barrier_stats = ShardStats(store); - eloqstore::FailPoint::GetInstance().ReleasePause(); - const bool bg_completed_while_armed = - wait_until([&] { return bg_done.load(std::memory_order_relaxed); }, - std::chrono::seconds(2)); - - stop_fg.store(true, std::memory_order_relaxed); + eloqstore::ReadRequest failed; + failed.SetArgs(tbl_id, key); + eloqstore::FailPoint::GetInstance().ArmOnce("KvTaskPageReadCqe"); + store->ExecSync(&failed); eloqstore::FailPoint::GetInstance().Disarm(); - for (std::thread &reader : readers) - { - reader.join(); - } - store->Stop(); - const eloqstore::IoQosStats stats = ShardStats(store); - CAPTURE(fg_primed, - bg_issued, - wake_gap_observed, - bg_incomplete_at_barrier, - barrier_stats.read_.inflight_, - primed_stats.read_.inflight_, - primed_stats.read_.blocked_count_); - REQUIRE(fg_primed); - REQUIRE(bg_issued); - REQUIRE(wake_gap_observed); - REQUIRE(bg_incomplete_at_barrier); - REQUIRE(barrier_stats.read_.inflight_ == 0); - REQUIRE(bg_completed_while_armed); - REQUIRE_FALSE(fg_failed.load(std::memory_order_relaxed)); - REQUIRE(bg_req.Error() == eloqstore::KvError::NoError); - REQUIRE(stats.read_.inflight_ == 0); - REQUIRE(stats.bg_read_.inflight_ == 0); - REQUIRE(stats.read_.high_watermark_ <= 1); - REQUIRE(stats.bg_read_.high_watermark_ <= 1); - REQUIRE(stats.bg_read_.blocked_count_ > 0); + REQUIRE(failed.Error() == eloqstore::KvError::IoFail); + REQUIRE(ShardStats(store).io_window_inflight_ == 0); + + eloqstore::ReadRequest recovery; + recovery.SetArgs(tbl_id, key); + store->ExecSync(&recovery); + REQUIRE(recovery.Error() == eloqstore::KvError::NoError); + REQUIRE(recovery.value_ == expected); + REQUIRE(ShardStats(store).io_window_inflight_ == 0); } -TEST_CASE("bg sub-budget: repeated ungated contention", "[io_qos][stress]") +TEST_CASE("io window: negative MergedWriteReq CQE releases and recovers", + "[io_qos]") { - eloqstore::KvOptions opts = default_opts; - opts.num_threads = 1; - opts.max_inflight_read = 2; - opts.bg_read_ratio = 50; - opts.overflow_pointers = 128; + // Same invariant for the merged-write cost (ceil(len/256KB) commands): + // acquire and release must stay symmetric on the error path. + eloqstore::KvOptions opts = append_opts; + opts.disk_rate_limit_iops = 0; + opts.max_inflight_io = 8; eloqstore::EloqStore *store = InitStore(opts); + const eloqstore::TableIdent tbl_id{"qos-merged-cqe", 0}; + const uint64_t ts = utils::UnixTs(); - const eloqstore::TableIdent fg_tbl{"qos-stress-fg", 0}; - MapVerifier fg_seed(fg_tbl, store, false); - fg_seed.SetValueSize(600 * 1024); - fg_seed.Upsert(0); - - const eloqstore::TableIdent bg_tbl{"qos-stress-bg", 0}; - MapVerifier bg_seed(bg_tbl, store, false); - bg_seed.SetValueSize(200); - bg_seed.Upsert(0, 2000); - - std::atomic failed{false}; - for (uint64_t round = 1; round <= 10; ++round) + eloqstore::BatchWriteRequest failed; + failed.SetTableId(tbl_id); + for (uint32_t i = 0; i < 400; ++i) { - std::atomic ready{0}; - std::atomic start{false}; - std::vector readers; - readers.reserve(8); - for (int i = 0; i < 8; ++i) - { - readers.emplace_back( - [&] - { - ready.fetch_add(1, std::memory_order_relaxed); - while (!start.load(std::memory_order_relaxed)) - { - std::this_thread::yield(); - } - for (int read = 0; read < 4; ++read) - { - eloqstore::ReadRequest req; - req.SetArgs(fg_tbl, test_util::Key(0, 7)); - store->ExecSync(&req); - if (req.Error() != eloqstore::KvError::NoError) - { - failed.store(true, std::memory_order_relaxed); - } - } - }); - } - while (ready.load(std::memory_order_relaxed) != readers.size()) - { - std::this_thread::yield(); - } - start.store(true, std::memory_order_relaxed); - - eloqstore::BatchWriteRequest bg_req; - bg_req.SetTableId(bg_tbl); - bg_req.AddWrite(test_util::Key(1000, 7), - std::string(200, 'b'), - utils::UnixTs() + round, + failed.AddWrite(test_util::Key(i, 7), + std::string(3000, 'm'), + ts, eloqstore::WriteOp::Upsert); - store->ExecSync(&bg_req); - if (bg_req.Error() != eloqstore::KvError::NoError) - { - failed.store(true, std::memory_order_relaxed); - } - for (std::thread &reader : readers) - { - reader.join(); - } } + eloqstore::FailPoint::GetInstance().ArmOnce("MergedWriteReqCqe"); + store->ExecSync(&failed); + eloqstore::FailPoint::GetInstance().Disarm(); - const eloqstore::IoQosStats stats = ShardStats(store); - REQUIRE_FALSE(failed.load(std::memory_order_relaxed)); - REQUIRE(stats.read_.inflight_ == 0); - REQUIRE(stats.bg_read_.inflight_ == 0); - REQUIRE(stats.read_.high_watermark_ <= 2); - REQUIRE(stats.bg_read_.high_watermark_ <= 1); - REQUIRE(stats.bg_read_.blocked_count_ > 0); + REQUIRE(failed.Error() == eloqstore::KvError::IoFail); + REQUIRE(ShardStats(store).io_window_inflight_ == 0); + + eloqstore::BatchWriteRequest recovery; + recovery.SetTableId(tbl_id); + recovery.AddWrite("recovery", "value", ts + 1, eloqstore::WriteOp::Upsert); + store->ExecSync(&recovery); + REQUIRE(recovery.Error() == eloqstore::KvError::NoError); + REQUIRE(ShardStats(store).io_window_inflight_ == 0); } TEST_CASE("io qos stats: concurrent sampling", "[io_qos][stats]") { + // IoQosStats must be safely sampleable from another thread while the + // shard runs (observability counters are relaxed atomics; the shard + // thread is the only writer). Ported from the count-budget suite. eloqstore::KvOptions opts = default_opts; - opts.max_inflight_read = 2; - opts.max_inflight_write = 8; + opts.disk_rate_limit_iops = 20000; + opts.max_inflight_io = 8; eloqstore::EloqStore *store = InitStore(opts); std::atomic stop{false}; @@ -788,30 +361,30 @@ TEST_CASE("io qos stats: concurrent sampling", "[io_qos][stats]") const eloqstore::IoQosStats stats = ShardStats(store); REQUIRE(samples.load(std::memory_order_relaxed) > 0); REQUIRE(active_samples.load(std::memory_order_relaxed) > 0); - REQUIRE(stats.read_.inflight_ == 0); - REQUIRE(stats.bg_read_.inflight_ == 0); - REQUIRE(stats.write_.inflight_ == 0); + REQUIRE(stats.rate_.admitted_ops_ > 0); + REQUIRE(stats.io_window_inflight_ == 0); } -TEST_CASE("io budgets: defaults are behavior-neutral", "[io_qos]") +TEST_CASE("rate budget: bytes bucket alone paces merged writes", "[io_qos]") { - // At default caps (read 64, write 32768) a single-threaded unit - // workload of small values must never block on a budget: foreground - // point reads are sequential (in-flight 1) and batch-write leaf loads - // are sequential background singles, well under bg_cap = 16. - eloqstore::EloqStore *store = InitStore(default_opts); + // Only the bytes bucket enabled (iops = 0): append-mode merged writes + // must charge bytes and complete. Exercises the ops-disabled branch of + // Positive() and the large-cost debt path (a merged write can exceed + // one burst of byte tokens). + eloqstore::KvOptions opts = append_opts; + opts.disk_rate_limit_mbps = 8; + opts.rate_limit_burst_ms = 4; + eloqstore::EloqStore *store = InitStore(opts); MapVerifier verify(test_tbl_id, store, false); - verify.SetValueSize(200); - verify.WriteRnd(0, 2000, 0, 25); - for (int i = 0; i < 100; i++) + verify.SetValueSize(1000); + verify.WriteRnd(0, 1000, 0, 25); + for (int i = 0; i < 20; i++) { - verify.Read(std::rand() % 2000); + verify.Read(std::rand() % 1000); } eloqstore::IoQosStats stats = ShardStats(store); - REQUIRE(stats.read_.inflight_ == 0); - REQUIRE(stats.write_.inflight_ == 0); - REQUIRE(stats.read_.blocked_count_ == 0); - REQUIRE(stats.write_.blocked_count_ == 0); + REQUIRE(stats.rate_.admitted_bytes_ > 0); + REQUIRE(stats.bg_rate_.admitted_bytes_ > 0); } From 8ebc95efdec2b4b5c3f7a0c818fb0148a5d18638 Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Mon, 3 Aug 2026 23:05:11 -0700 Subject: [PATCH 25/30] fix(io): address M4 rate-limiter review (currency, overflow, config, docs) Review findings on the M4 commit, verified against the code and fixed: - Write currency is the physical 4KB page quantum, not 2KB. The old SubmitMergedWrite clamp (max(rate_limit_io_unit, data_page_size)) silently floored the 2KB default at 4KB, so the "2KB accounting" never ran; the read-tail target was met at the effective 4KB. Default and docs now say 4KB, the clamp is removed, and one WriteRateOps helper meters WritePage and SubmitMergedWrite (unit-tested 4KB->1, 1MB->256). - RateBudget::Charge debits a balance only when its dimension is enabled. Previously a disabled dimension (e.g. the bytes bucket at the default disk_rate_limit_mbps=0) was debited every IO but never refilled, driving it to int64 overflow (~9TB) on a long-lived shard. - SetRates splits via SplitRate, keeping both class rates nonzero for any enabled dimension so integer truncation at low per-shard rates cannot silently disable a class (unbudgeted background writes / lost read protection). - Pure-write workloads run at rate_bg_ratio of the device rate by design (all writes are background, no reverse lending); documented as an accepted product decision in io_qos.md and 08-data-lifecycle.md. - benchmark/opts_interference.ini and the committed acceptance path now drive the M4 knobs, not the deprecated count knobs (which changed nothing). GET2 exits nonzero on request failures or zero samples. - Malformed M4 options fall back to their member defaults (burst_ms) and reject a zero io_unit rather than silently changing policy. - The retired per-class in-flight page gauges are unregistered rather than reporting the class-blind command window mislabeled as read pages. - io_qos.cpp: bytes-only test disables IOPS to actually cover the ops-disabled path; the borrow test forces demand with a concurrent overflow read instead of a device-speed-dependent sequential stream. - Architecture docs (07-io-stack, 02-runtime-and-lifecycle, 04-execution-model, 08-data-lifecycle) and the io_qos.md summary now describe the M4 RateBudget instead of the retired M1/M2 count budgets. Full local suite green (325/325). Co-Authored-By: Claude Opus 4.8 --- benchmark/eloq_store_bm.cc | 12 ++ benchmark/eloq_store_bm.h | 11 ++ benchmark/main.cpp | 7 +- benchmark/opts_interference.ini | 24 ++-- docs/architecture/02-runtime-and-lifecycle.md | 18 +-- docs/architecture/04-execution-model.md | 29 ++--- docs/architecture/07-io-stack.md | 58 +++++----- docs/architecture/08-data-lifecycle.md | 10 +- docs/design/io_qos.md | 81 ++++++++----- include/async_io_manager.h | 19 ++++ include/kv_options.h | 29 +++-- src/async_io_manager.cpp | 106 +++++++++++++----- src/eloq_store.cpp | 13 +-- src/kv_options.cpp | 18 ++- src/storage/shard.cpp | 15 ++- tests/io_qos.cpp | 59 +++++++--- 16 files changed, 333 insertions(+), 176 deletions(-) diff --git a/benchmark/eloq_store_bm.cc b/benchmark/eloq_store_bm.cc index 0d3278fd7..99f0397dc 100644 --- a/benchmark/eloq_store_bm.cc +++ b/benchmark/eloq_store_bm.cc @@ -773,6 +773,18 @@ void Benchmark::RunGet2(uint32_t client_threads, << ", p50->" << pct(0.50) << ", p90->" << pct(0.90) << ", p95->" << pct(0.95) << ", p99->" << pct(0.99) << ", p99.9->" << pct(0.999) << ", p99.99->" << pct(0.9999); + + // A latency benchmark that reports percentiles over surviving samples + // must not exit success when requests were rejected/failed or nothing + // completed — otherwise a broken run reads as a fast one. main() + // turns this into a nonzero process exit. + if (read_failures != 0 || issue_failures != 0 || successes == 0) + { + failed_ = true; + LOG(ERROR) << "GET2 run FAILED: read_failures=" << read_failures + << " issue_failures=" << issue_failures + << " successes=" << successes; + } } void Benchmark::OnRead(::eloqstore::KvRequest *req) diff --git a/benchmark/eloq_store_bm.h b/benchmark/eloq_store_bm.h index 0a2a904d6..1176482a5 100644 --- a/benchmark/eloq_store_bm.h +++ b/benchmark/eloq_store_bm.h @@ -226,6 +226,16 @@ class Benchmark uint32_t per_shard_cap); static void OnReadV2(::eloqstore::KvRequest *req); + /** + * @brief True if the last run recorded request failures or produced no + * successful samples. main() propagates it to a nonzero process exit so + * a broken run cannot masquerade as a fast one. + */ + bool Failed() const + { + return failed_; + } + private: static void OnBatchWrite(::eloqstore::KvRequest *req); @@ -260,6 +270,7 @@ class Benchmark mutable std::vector load_obj_gens_; uint64_t start_ts_{0}; mutable BMResult result_; + bool failed_{false}; // set by RunGet2 on request failure / no samples friend BMResult; friend LoadPartitionsOperation; diff --git a/benchmark/main.cpp b/benchmark/main.cpp index 988e3a40b..46545e2b7 100644 --- a/benchmark/main.cpp +++ b/benchmark/main.cpp @@ -86,6 +86,7 @@ DEFINE_uint64(subcompactions, int main(int argc, char *argv[]) { + int exit_code = 0; FLAGS_logtostderr = true; google::InitGoogleLogging("EloqStore_benchmark"); @@ -141,6 +142,10 @@ int main(int argc, char *argv[]) // Run the test bench_mark.RunBenchmark(); + if (bench_mark.Failed()) + { + exit_code = 1; + } // Shutdown the eloq store bench_mark.CloseEloqStore(); @@ -196,5 +201,5 @@ int main(int argc, char *argv[]) } google::ShutdownGoogleLogging(); - return 0; + return exit_code; } \ No newline at end of file diff --git a/benchmark/opts_interference.ini b/benchmark/opts_interference.ini index 1a5ab4097..43d6b28c4 100644 --- a/benchmark/opts_interference.ini +++ b/benchmark/opts_interference.ini @@ -1,12 +1,18 @@ -# EloqStore options for interference_bench (docs/design/io_qos.md commit 3). +# EloqStore options for interference_bench (docs/design/io_qos.md M4). # # Append mode with small (8MB) data files so the rotating partial-overwrite # storm pushes per-file space amplification past file_amplify_factor quickly, # keeping compaction's move pipeline (background ReadPages bursts) busy. # -# Sweep the QoS knobs below (see io_qos_impl_plan.md "Performance -# acceptance"): max_inflight_read in {64,128,256,512}, bg_read_ratio in -# {10,25,50}, max_inflight_write in {256,512,1024}. +# Sweep the M4 device rate-limit knobs below (see docs/design/io_qos.md +# "Sizing contract" and "Validation plan"): +# disk_rate_limit_iops in {200000, 260000, 275000, 290000} (~95% of the +# fio-measured device ceiling is the recommended setting) +# rate_bg_ratio in {10, 25, 50} (background share of the budget) +# rate_limit_burst_ms in {1, 2, 4} +# A/B the limiter by setting disk_rate_limit_iops = 0 (off) versus a +# calibrated value; the deprecated count knobs (max_inflight_read, +# bg_read_ratio, max_inflight_write) no longer affect device admission. # Note: run interference_bench with --partitions >= num_threads, or some # shards sit idle and their RESULT qos lines read zero. @@ -18,10 +24,12 @@ fd_limit = 5000 num_retained_archives = 0 skip_verify_checksum = true -# --- IO QoS knobs under test --- -max_inflight_read = 64 -bg_read_ratio = 25 -max_inflight_write = 512 +# --- M4 device rate limit under test --- +disk_rate_limit_iops = 275000 +rate_bg_ratio = 25 +rate_limit_burst_ms = 2 +rate_limit_io_unit = 4KB +# max_inflight_io = 0 # optional class-blind in-flight window [permanent] store_path = /tmp/eloqstore_interference diff --git a/docs/architecture/02-runtime-and-lifecycle.md b/docs/architecture/02-runtime-and-lifecycle.md index eb3f628f3..192153717 100644 --- a/docs/architecture/02-runtime-and-lifecycle.md +++ b/docs/architecture/02-runtime-and-lifecycle.md @@ -107,14 +107,16 @@ state: `enable_data_page_cache`), `root_meta_cache_size` (global RootMeta LRU). - **Write shaping**: `max_write_concurrency`, `write_buffer_size`/`write_buffer_ratio` (append-mode aggregation), - `manifest_limit` (snapshot-rotation threshold). - (`max_write_batch_pages` is deprecated and ignored — superseded by the - `max_inflight_write` IO budget, doc 07.) -- **IO QoS** (doc 07, `docs/design/io_qos.md`): `max_inflight_read` - (device-calibrated read queue-depth cap), `bg_read_ratio` (background - read sub-budget, the tail-predictability policy knob), - `max_inflight_write` (write queue-depth cap; also sizes the write - request pools). + `manifest_limit` (snapshot-rotation threshold), `max_inflight_write` + (write request-pool sizing). + (`max_write_batch_pages` is deprecated and ignored.) +- **IO QoS** (doc 07, `docs/design/io_qos.md` M4): `disk_rate_limit_iops` + /`disk_rate_limit_mbps` (per-disk device rate limit, on by default, + divided across shards), `rate_bg_ratio` (background share of the rate — + the tail-predictability policy knob), `rate_limit_burst_ms`, + `rate_limit_io_unit` (write ops quantum), and `max_inflight_io` (optional + class-blind in-flight command window). The former count budgets + `max_inflight_read`/`bg_read_ratio` are deprecated no-ops. - **Cloud**: `cloud_provider/endpoint/region/keys`, `local_space_limit` + `reserve_space_ratio` (cache budget), `max_cloud_concurrency`, `cloud_request_threads`, `allow_reuse_local_caches`, `prewarm_cloud_cache`. diff --git a/docs/architecture/04-execution-model.md b/docs/architecture/04-execution-model.md index 94b3f703f..d1d959ca2 100644 --- a/docs/architecture/04-execution-model.md +++ b/docs/architecture/04-execution-model.md @@ -59,20 +59,21 @@ Scheduling primitives: - `WaitingZone` / `WaitingSeat` / `Mutex` — intra-shard wait lists (no real locks; they park/wake coroutines). Used for FD open/close exclusion, pool exhaustion waits, upload completion, etc. -- IO-budget waits (`IoBudget::Acquire`, `async_io_manager.h`) — tasks park on - a budget's `WaitingZone` when admitting their page IO would exceed the - shard's in-flight read/write cap (`max_inflight_read` / - `max_inflight_write`); `PollComplete` releases per CQE and wakes waiters, - so release never depends on the blocked task being scheduled. Background - tasks (`KvTask::IsBackground()`: BatchWrite, BackgroundWrite, EvictFile, - Prewarm) are additionally confined to a read sub-budget (`bg_read_ratio`) - and wait on a separate FIFO zone. From a background acquisition's first wait - through admission (including the wake-to-admit gap), its unused sub-budget is - reserved from new foreground admissions. Release wakes background first and - always wakes foreground; both classes re-check admission, so neither can - starve the other. See `docs/design/io_qos.md` (M1/M2); the acquire order is - FD/mutex → pools/buffers → budget → SQE, with no voluntary yield after budget - admission and an equal-cost release per CQE. +- Rate-budget waits (`RateBudget::Acquire`, `async_io_manager.h`) — tasks park + on a per-class `WaitingZone` when admitting their page IO would drive the + shard's device rate budget (`disk_rate_limit_iops`/`disk_rate_limit_mbps`, + M4) non-positive. Wakes are **refill-driven**, not completion-driven: the + once-per-loop `RefillAndWake` (peek-and-grant — it charges the FIFO head's + recorded cost before waking it) is the only wake source, so waiter progress + depends only on the shard loop running, never on another task completing. + Background tasks (`KvTask::IsBackground()`: BatchWrite, BackgroundWrite, + EvictFile, Prewarm) draw from the background class share (`rate_bg_ratio`) + on a separate FIFO zone; foreground may borrow background's idle surplus but + background never borrows foreground's, so foreground's share is a hard + guarantee. The optional `max_inflight_io` window is the only occupancy cap + that releases per CQE. See `docs/design/io_qos.md` (M4); the acquire order is + FD/mutex → pools/buffers → rate budget → window → SQE, with no voluntary + yield after admission. (The former `IoBudget` count budgets are retired.) `TaskManager` keeps one free-list pool per task type (`BatchWriteTask`, `BackgroundWrite`, `ReadTask`, `ScanTask`, `ListObjectTask`, diff --git a/docs/architecture/07-io-stack.md b/docs/architecture/07-io-stack.md index 84165c45a..9e8a8aed1 100644 --- a/docs/architecture/07-io-stack.md +++ b/docs/architecture/07-io-stack.md @@ -37,38 +37,32 @@ Responsibilities: - **Page I/O** — `ReadPage`/`ReadPages` (batched, into pool buffers, fixed reads when the buffer is registered), `WritePage`. `ConvFilePageId` splits a `FilePageId` into `(file_id, offset)` by `pages_per_file_shift`. -- **In-flight page-IO budgets** (`IoBudget`, see `docs/design/io_qos.md` - M1/M2) — two per-shard counters in configured `data_page_size` units with - independent caps: - `read_budget_` (`max_inflight_read`; `ReadPage`/`ReadPages`, per-page - acquisition so a batch larger than the cap cannot deadlock) and - `write_budget_` (`max_inflight_write`; `WritePage` cost 1, - `SubmitMergedWrite` cost `ceil(bytes / data_page_size)`). Budget is acquired - immediately before SQE prep and released per CQE in `PollComplete`, which - distinguishes budgeted page reads from metadata ops via the - `KvTaskPageRead`/`BaseReqPageRead` user-data types. A cap of 0 disables a - budget; a single request costlier than the cap is admitted alone once the - budget drains. Metadata, manifest, bulk file/snapshot paths (`ReadFile`, - `ReadFilePrefix`, `WriteSnapshot`), `Fdatasync`, and segment IO are exempt. - With `enable_data_page_cache`, `max_inflight_write` also bounds cached-page - pins retained by write promotion until the corresponding IO completes. - The read budget carries a **background sub-budget** (`bg_read_ratio` - percent of `max_inflight_read`): budgeted page reads from `BatchWrite` and - `BackgroundWrite` (compaction) tasks are additionally bounded by it, so they - cannot crowd foreground point reads out of the device queue. `EvictFile` and - `Prewarm` are background task types, but local-GC `ReadFile` and - prewarm/download whole-file bulk IO remain exempt. Foreground may use the - entire read budget while background has no pending demand. Once a background - acquisition enters the wait path, its unused sub-budget stays reserved - through admission, including the wake-to-admit gap. Each class waits on its - own FIFO zone; release wakes background first and always wakes foreground. - The write budget has no split — all page writes come from write tasks, i.e. - background. - `GetIoQosStats()` (also surfaced as `EloqStore::GetIoQosStats(shard_id)`) - exposes in-flight/high-watermark/blocked counters (total read, bg-read slice, - write) plus write-path fdatasync count and latency. The blocked fields are - per admission class: `read_` counts foreground waits, `bg_read_` background - waits, and `write_` all write waits. +- **Device rate budget** (`RateBudget`, see `docs/design/io_qos.md` M4) — + per-shard token buckets metering device **ops/sec** and **bytes/sec**, + refilled lazily from the shard TSC clock once per event-loop iteration + (`RefillAndWake` at the top of `Submit()`) and spent at the page-IO acquire + sites (`ReadPage`/`ReadPages` cost 1 op + one page of bytes; `WritePage` and + `SubmitMergedWrite` cost `WriteRateOps(bytes)` = `ceil(bytes / + rate_limit_io_unit)` ops + `bytes`; `fdatasync` 1 op). Admission is **debt**: + a task waits until the balance is positive, then subtracts its full cost + (may go negative), so the long-run rate is exact and an IO larger than the + bucket never deadlocks. There is **no completion-time release** — a rate + budget meters issuance, so tokens are spent by sending and only time (refill) + restores them. The budget is **partitioned by class**: foreground buckets + refill at `(100 − rate_bg_ratio)%`, background (background-task reads and all + write-path IO) at `rate_bg_ratio%`. Foreground may borrow background's idle + surplus (background never borrows); admission is peek-and-grant — the refill + charges the FIFO head's recorded cost before waking it. Per-disk limits + (`disk_rate_limit_iops`/`disk_rate_limit_mbps`, on by default) are divided + across shards (`× store paths / num_threads`). A separate optional + class-blind in-flight command window (`max_inflight_io`, off by default) is + the only occupancy-style cap and *does* release per CQE in `PollComplete`. + Metadata, manifest, bulk file/snapshot paths, and segment IO are exempt. + `GetIoQosStats()` (also `EloqStore::GetIoQosStats(shard_id)`) exposes the + per-class rate counters (blocked count/us, admitted ops/bytes, borrowed ops) + and the window's in-flight/high-water/blocked gauges. + The M1/M2 count budgets (`IoBudget`, `max_inflight_read`/`bg_read_ratio`) + are **retired**; `max_inflight_write` now only sizes the write request pools. - **FD cache** — `LruFD` per (partition, `TypedFileId`), doubly-linked LRU bounded by the shard's fd budget; `EvictFD` closes idle descriptors. Open/close exclusion per FD via a coroutine `Mutex`. Data files open with diff --git a/docs/architecture/08-data-lifecycle.md b/docs/architecture/08-data-lifecycle.md index fd3fb754c..bf6a5baa8 100644 --- a/docs/architecture/08-data-lifecycle.md +++ b/docs/architecture/08-data-lifecycle.md @@ -68,10 +68,12 @@ user writes. Scheduled via shard pending-sets requests in each `PendingWriteQueue`. All of these run as background tasks (`KvTask::IsBackground()`), so their -data-page reads — compaction move batches in particular — are charged -against the read budget's background sub-budget (`bg_read_ratio`, -doc 07 / `docs/design/io_qos.md` M2) and cannot crowd out foreground reads -at the device; their page writes are bounded by `max_inflight_write`. +data-page reads — compaction move batches in particular — and all their page +writes are charged against the **background share** of the device rate budget +(`rate_bg_ratio`, doc 07 / `docs/design/io_qos.md` M4) and cannot crowd out +foreground reads at the device. A pure-write/compaction phase with no +concurrent foreground reads therefore runs at `rate_bg_ratio` of the device +rate by design (foreground does not lend to background). - **Compaction** (`Compact()`, append mode only): one `MakeCowRoot`; rewrite pass over under-utilized data files (live pages re-written to the tail, diff --git a/docs/design/io_qos.md b/docs/design/io_qos.md index 548e0e73d..c9228cea8 100644 --- a/docs/design/io_qos.md +++ b/docs/design/io_qos.md @@ -22,18 +22,23 @@ device level. Reads were not budgeted at all; writes were budgeted only per-task. Yielding controlled when background IO was *issued*, not how much of it queued at the device once issued. -This document defines three per-shard mechanisms. M1 and M2 are implemented; -M3 remains a measurement-gated follow-up: - -- **M1**: per-shard caps on in-flight page IO, with **separate caps for - reads and writes** — they are different device resources and must be - tunable independently. -- **M2**: foreground/background classification with a background sub-budget - on the read cap (all page writes come from write tasks, i.e. background, - so the write cap needs no split). -- **M3**: a bytes/sec rate limiter on background writes (follow-up, driven by - measurement; complements the write cap, which bounds queue depth but not - sustained throughput). +**Current state (2026):** the shipped mechanism is **M4**, per-shard device +rate limiting (`RateBudget`; see the M4 section below). M1/M2 (in-flight +count budgets) and M3 (a separate background write bytes/sec limiter) were +the original design and are documented below as design record, but M1/M2 are +**retired** in the code and M3 is **subsumed** by M4's byte bucket. The +historical mechanism sketch: + +- **M1** *(retired)*: per-shard caps on in-flight page IO, separate for reads + and writes. +- **M2** *(retired; the FG/BG class model survives in M4's `rate_bg_ratio`)*: + foreground/background classification with a background sub-budget on the + read cap. +- **M3** *(subsumed by M4)*: a bytes/sec rate limiter on background writes. +- **M4** *(current)*: per-shard token buckets metering device ops and bytes + per second, partitioned foreground/background, replacing M1/M2 — the tail + on rate-metered cloud disks is set by the hypervisor's rate limiter, which + a concurrency cap cannot address. All caps and limits in this document are **per shard**, consistent with the rest of EloqStore. @@ -342,8 +347,19 @@ M2 class policy: (measured ~2M borrowed ops/shard per storm; foreground fell 183K → 116K QPS and p99.9 720 µs → 5.6 ms). Foreground's share is a guarantee against background and must hold regardless of how idle foreground - momentarily looks. Reverse lending (idle foreground donating to - background) would need genuine idle-hysteresis to be safe; deferred. + momentarily looks. **Accepted consequence (product decision):** because + all write-path IO is background and reverse lending is not implemented, a + pure-write workload (no concurrent foreground reads) runs at only + `rate_bg_ratio` of the device rate — 25% by default — even when the + device is otherwise idle. This is deliberate: the limiter is on by + default and read-tail protection takes priority; pure-write throughput is + no longer a ±3% no-regression guard. Deployments that need full-rate + ingest raise `rate_bg_ratio` (or disable the limiter for a load phase). + Reverse lending (idle foreground donating to background) would remove the + cap but needs genuine idle-hysteresis to stay safe — the instantaneous + "foreground idle?" test that works for foreground borrowing is unsafe in + reverse (a closed-loop foreground looks idle for microseconds between + completions); deferred. Each refill wakes each class's zone independently — no cross-class wake coupling, hence no starvation coupling either. @@ -356,12 +372,13 @@ M2 class policy: There is no completion-driven release — spent tokens are gone; the refill is the only credit source. -**Relation to M1/M2.** The count caps remain as burst-depth guards (a rate -bucket alone would admit `burst_ms` worth of IO instantaneously after an idle -gap). Their sizing pressure disappears: set them to `2 × shard_iops × -t_read(loaded)` and forget them; the rate budget is the binding control. M3 -(background write bytes/sec) becomes the BG class share of the byte bucket — -no separate mechanism. +**Relation to M1/M2.** The M1/M2 count budgets are **retired** (`IoBudget` +deleted; `max_inflight_read`/`bg_read_ratio` are deprecated no-ops; +`max_inflight_write` reverts to write request-pool sizing). Instantaneous +burst-depth bounding, if a deployment wants it, is the single class-blind +`max_inflight_io` window (off by default; measured inert on the rate-metered +Azure disk). M3 (background write bytes/sec) is subsumed: background write +pacing is the BG class share of the M4 byte bucket — no separate mechanism. **Recommended production configuration (validated 2026-07-22): the rate knobs alone — the count caps are retired.** With the partitioned buckets, @@ -370,9 +387,10 @@ the tuned count caps changed storm p99.9 by nothing measurable (709 vs `disk_rate_limit_iops` (95% of the measured disk ceiling; default 275K — the measured Azure v2 local-NVMe ceiling — with rate limiting ON by default), `rate_bg_ratio` (the one policy choice), and -`rate_limit_io_unit` (a platform constant — 2 KB on Azure local NVMe, -i.e. written bytes cost twice read bytes per 4 KB; values ≥16 KB -measurably leak hypervisor throttling, 4 KB is equivalent to 2 KB). +`rate_limit_io_unit` (the physical write quantum, default 4 KB = one data +page: a 4 KB write costs 1 op, a 1 MB merged write 256; a finer unit +charges writes more ops and paces background harder — not needed at the +tested Azure read-tail target, so left at 4 KB). `max_inflight_io` (single class-blind in-flight command window) exists as an off-by-default safety bound; measured inert on Azure — the rate-metered hypervisor does not penalize instantaneous depth at burst-window @@ -399,14 +417,15 @@ uint32_t rate_limit_burst_ms = 2; // bucket capacity, ms of refill; // flattens the distribution (median // up, tail down), larger the reverse. // 1 ms for tail-first deployments. -uint32_t rate_limit_io_unit = 2048; // ops-cost quantum for large IOs: - // the hypervisor ACCOUNTING currency - // (Azure: a written 4KB costs two - // read units, fio-fitted and - // ladder-confirmed; >=16KB leaks - // throttling). Overcharges writes on - // platforms with cheaper accounting - // — the safe direction. +uint32_t rate_limit_io_unit = 4096; // write ops quantum: ceil(len/unit) + // ops per write (WriteRateOps, used by + // both WritePage and SubmitMergedWrite). + // Default = one data page: 4KB write = + // 1 op, 1MB merged = 256. Must be + // nonzero (validated at load). A finer + // unit paces background harder; not + // needed at the tested read-tail + // target. uint32_t rate_bg_ratio = 25; // background share of the rate, percent ``` diff --git a/include/async_io_manager.h b/include/async_io_manager.h index 179f7a59b..836dede3a 100644 --- a/include/async_io_manager.h +++ b/include/async_io_manager.h @@ -1398,6 +1398,25 @@ class IouringMgr : public AsyncIoManager kDeviceCmdBytes); } + /** + * @brief Rate-budget ops cost of a write of @p bytes, in + * rate_limit_io_unit-sized device-accounting operations + * (ceil(bytes / unit)). The single source of the write ops charge — + * both WritePage (one data page) and SubmitMergedWrite use it, so a + * page's cost and a merged write's cost are consistent by construction. + * The unit is the configured physical write quantum (default = data + * page size); it is validated nonzero at option load, so no clamp is + * needed here. + */ + static uint32_t WriteRateOpsFor(size_t bytes, uint32_t unit) + { + return static_cast((bytes + unit - 1) / unit); + } + uint32_t WriteRateOps(size_t bytes) const + { + return WriteRateOpsFor(bytes, options_->rate_limit_io_unit); + } + /** * @brief Pure-function form of BufIndexForAddress: searches @p chunks for * the one containing @p ptr and returns base_index + chunk_index, or diff --git a/include/kv_options.h b/include/kv_options.h index eb37c02b2..3349d984e 100644 --- a/include/kv_options.h +++ b/include/kv_options.h @@ -135,21 +135,20 @@ struct KvOptions */ uint32_t rate_limit_burst_ms = 2; /** - * @brief Ops-cost quantum for large IOs against the ops bucket (M4): - * a single submission of `len` bytes is charged - * ceil(len / rate_limit_io_unit) operations. This is the HYPERVISOR - * ACCOUNTING currency, not the kernel command split: on Azure local - * NVMe, written bytes cost two read units per 4KB (fio-fitted - * 2026-07-21/22, confirmed by the clean/throttled boundary and the - * engine currency ladder — 4KB units behaved identically to 2KB, - * >=16KB measurably re-exposed hypervisor throttling under write - * load). The 2KB default encodes that price. On platforms with - * different accounting it errs in the safe direction (overcharging - * writes paces background early instead of blowing the foreground - * tail); recalibrate with the fio boundary method in - * docs/design/io_qos.md if write throughput matters more. - */ - uint32_t rate_limit_io_unit = 2 * KB; + * @brief Ops-cost quantum for writes against the ops bucket (M4): a + * write of `len` bytes is charged ceil(len / rate_limit_io_unit) + * operations (WriteRateOps), so a data-page write and a merged write + * are metered on one consistent scale. Defaults to the physical write + * quantum, one data page (4KB): a 4KB write costs 1 op, a 1MB merged + * write costs 256. Must be nonzero (a zero/malformed value is rejected + * at load and the default is kept). A finer unit (e.g. 2KB) charges + * writes more ops per byte and paces background harder; the tested + * Azure NVMe read-tail target was met at the 4KB default, so it is not + * enabled by default — recalibrate with the fio boundary method in + * docs/design/io_qos.md only if a device's write-accounting unit is + * measured smaller and write throttling needs it. + */ + uint32_t rate_limit_io_unit = 4 * KB; /** * @brief Background share of the device rate budget, percent (M4). * The rate budget is partitioned: background IO (background-task diff --git a/src/async_io_manager.cpp b/src/async_io_manager.cpp index 522528425..121ed4ff7 100644 --- a/src/async_io_manager.cpp +++ b/src/async_io_manager.cpp @@ -167,19 +167,50 @@ bool AsyncIoManager::IsIdle() return true; } +// Split a per-shard total rate into (foreground, background) shares. A +// rate of 0 means "dimension disabled" and is preserved as such. For any +// enabled dimension both shares are kept nonzero so integer truncation at +// low totals cannot silently disable a class — a zero background rate +// would leave background writes unbudgeted, a zero foreground rate would +// drop read protection. The sum equals the total exactly for total >= 2; +// a total of 1 cannot fund two integer shares, so both are set to 1 (a +// one-unit overshoot at a degenerate rate) and logged. +static void SplitRate(uint64_t total, + uint32_t bg_ratio_pct, + const char *dim, + uint64_t &fg, + uint64_t &bg) +{ + if (total == 0) + { + fg = 0; + bg = 0; + return; + } + if (total < 2) + { + LOG(WARNING) << "per-shard " << dim << " rate " << total + << " too low to partition into foreground/background; " + "using 1/1 to keep both classes nonzero"; + fg = 1; + bg = 1; + return; + } + bg = std::clamp(total * bg_ratio_pct / 100, 1, total - 1); + fg = total - bg; +} + void RateBudget::SetRates(uint64_t ops_per_sec, uint64_t bytes_per_sec, uint32_t burst_ms, uint32_t bg_ratio_pct) { // Partition the total rate between the classes (see class comment): - // background gets ratio percent, foreground the rest. Clamped so both - // classes always have a nonzero share when the budget is enabled. + // background gets ratio percent, foreground the rest, both kept nonzero + // for any enabled dimension (see SplitRate). const uint32_t ratio = std::clamp(bg_ratio_pct, 1, 99); - fg_ops_rate_ = ops_per_sec * (100 - ratio) / 100; - fg_bytes_rate_ = bytes_per_sec * (100 - ratio) / 100; - bg_ops_rate_ = ops_per_sec * ratio / 100; - bg_bytes_rate_ = bytes_per_sec * ratio / 100; + SplitRate(ops_per_sec, ratio, "ops", fg_ops_rate_, bg_ops_rate_); + SplitRate(bytes_per_sec, ratio, "bytes", fg_bytes_rate_, bg_bytes_rate_); burst_us_ = uint64_t{std::max(burst_ms, 1)} * 1000; } @@ -267,7 +298,10 @@ void RateBudget::Charge(uint32_t ops, uint64_t bytes, bool background) { // Debt semantics: subtract the full cost, letting the balance go // negative. Callers guarantee CanAdmit(background) held at the moment - // of the charge. + // of the charge. A balance is debited ONLY when its dimension is + // enabled (rate != 0): a disabled dimension is never refilled and + // never consulted by Positive(), so debiting it would drive it + // monotonically negative to int64 overflow on a long-lived shard. const int64_t ops_cost = int64_t{ops} * kScale; const int64_t bytes_cost = static_cast(bytes) * kScale; if (background) @@ -277,8 +311,14 @@ void RateBudget::Charge(uint32_t ops, uint64_t bytes, bool background) // background buckets — the foreground debit is unreachable by // construction, not by luck. assert(Positive(true)); - bg_ops_bal_ -= ops_cost; - bg_bytes_bal_ -= bytes_cost; + if (bg_ops_rate_ != 0) + { + bg_ops_bal_ -= ops_cost; + } + if (bg_bytes_rate_ != 0) + { + bg_bytes_bal_ -= bytes_cost; + } bg_admitted_ops_.fetch_add(ops, std::memory_order_relaxed); bg_admitted_bytes_.fetch_add(bytes, std::memory_order_relaxed); return; @@ -289,13 +329,25 @@ void RateBudget::Charge(uint32_t ops, uint64_t bytes, bool background) const bool borrowed = !Positive(false); if (borrowed) { - bg_ops_bal_ -= ops_cost; - bg_bytes_bal_ -= bytes_cost; + if (bg_ops_rate_ != 0) + { + bg_ops_bal_ -= ops_cost; + } + if (bg_bytes_rate_ != 0) + { + bg_bytes_bal_ -= bytes_cost; + } } else { - fg_ops_bal_ -= ops_cost; - fg_bytes_bal_ -= bytes_cost; + if (fg_ops_rate_ != 0) + { + fg_ops_bal_ -= ops_cost; + } + if (fg_bytes_rate_ != 0) + { + fg_bytes_bal_ -= bytes_cost; + } } admitted_ops_.fetch_add(ops, std::memory_order_relaxed); admitted_bytes_.fetch_add(bytes, std::memory_order_relaxed); @@ -1303,12 +1355,14 @@ KvError IouringMgr::WritePage(const TableIdent &tbl_id, auto [fd, registered] = fd_ref.FdPair(); WriteReq *req = write_req_pool_->Alloc(std::move(fd_ref), std::move(page)); - // Write-budget admission (io_qos.md M1): after every other blocking - // acquisition (FD, req pool), immediately before SQE prep. The rate - // budget (M4) is charged first; write tasks classify as background, - // so page writes draw from the background sub-bucket. - rate_budget_.Acquire( - 1, options_->data_page_size, ThdTask()->IsBackground()); + // Device admission (io_qos.md M4): after every other blocking + // acquisition (FD, req pool), immediately before SQE prep. Rate budget + // first; write tasks classify as background. The ops cost uses the same + // WriteRateOps helper as the merged-write path so a page and a merged + // write are charged on one consistent scale. + rate_budget_.Acquire(WriteRateOps(options_->data_page_size), + options_->data_page_size, + ThdTask()->IsBackground()); AcquireIoWindow(1); io_uring_sqe *sqe = GetSQE(UserDataType::WriteReq, req); if (registered) @@ -1615,18 +1669,12 @@ KvError IouringMgr::SubmitMergedWrite(const TableIdent &tbl_id, static_cast(req->pages_.size() - 1); } - // Write-budget admission (io_qos.md M1): cost in 4KB-page units so the - // cap means the same thing in append and non-append mode. Must mirror - // the release cost computed from bytes_ in PollComplete. - // Rate budget (M4) first: ops cost mirrors the kernel's split of large - // IOs into device commands of at most rate_limit_io_unit bytes; the + // Device admission (io_qos.md M4). Rate budget first: ops cost is + // ceil(bytes / rate_limit_io_unit) via WriteRateOps (the same helper + // WritePage uses), charged against the configured write quantum; the // bytes bucket is charged the full length. Debt admission means this // single large acquisition never deadlocks against the bucket size. - const uint32_t io_unit = std::max(options_->rate_limit_io_unit, - options_->data_page_size); - rate_budget_.Acquire(static_cast((bytes + io_unit - 1) / io_unit), - bytes, - ThdTask()->IsBackground()); + rate_budget_.Acquire(WriteRateOps(bytes), bytes, ThdTask()->IsBackground()); AcquireIoWindow(DeviceCmdCost(bytes)); io_uring_sqe *sqe = GetSQE(UserDataType::MergedWriteReq, req); auto [fd, registered] = req->fd_ref_.FdPair(); diff --git a/src/eloq_store.cpp b/src/eloq_store.cpp index 7a03a5def..0509b627e 100644 --- a/src/eloq_store.cpp +++ b/src/eloq_store.cpp @@ -2371,13 +2371,12 @@ void EloqStore::InitializeMetrics(metrics::MetricsRegistry *metrics_registry, metrics::Type::Gauge); metrics_meters_[i]->Register(metrics::NAME_ELOQSTORE_LOCAL_SPACE_LIMIT, metrics::Type::Gauge); - metrics_meters_[i]->Register( - metrics::NAME_ELOQSTORE_INFLIGHT_READ_PAGES, metrics::Type::Gauge); - metrics_meters_[i]->Register( - metrics::NAME_ELOQSTORE_INFLIGHT_BG_READ_PAGES, - metrics::Type::Gauge); - metrics_meters_[i]->Register( - metrics::NAME_ELOQSTORE_INFLIGHT_WRITE_PAGES, metrics::Type::Gauge); + // The per-class in-flight page gauges (read / bg-read / write) + // measured the retired M1/M2 count budgets. The M4 rate budget has + // no per-class instantaneous page depth to report (it meters + // cumulative ops/bytes and a class-blind command window), so these + // gauges are not registered; dedicated rate-budget metrics are a + // follow-up (docs/design/io_qos.md). } enable_eloqstore_metrics_ = true; diff --git a/src/kv_options.cpp b/src/kv_options.cpp index 200661d2a..6e52d80ee 100644 --- a/src/kv_options.cpp +++ b/src/kv_options.cpp @@ -180,13 +180,25 @@ int KvOptions::LoadFromIni(const char *path) } if (reader.HasValue(sec_run, "rate_limit_burst_ms")) { - rate_limit_burst_ms = - reader.GetUnsigned(sec_run, "rate_limit_burst_ms", 4); + // Fall back to the current (member-default) value, not a literal, so + // a malformed entry cannot silently change the policy. + rate_limit_burst_ms = reader.GetUnsigned( + sec_run, "rate_limit_burst_ms", rate_limit_burst_ms); } if (reader.HasValue(sec_run, "rate_limit_io_unit")) { std::string io_unit_str = reader.Get(sec_run, "rate_limit_io_unit", ""); - rate_limit_io_unit = ParseSizeWithUnit(io_unit_str); + const uint64_t parsed = ParseSizeWithUnit(io_unit_str); + if (parsed == 0) + { + LOG(WARNING) << "rate_limit_io_unit '" << io_unit_str + << "' is invalid; keeping default " + << rate_limit_io_unit << " bytes"; + } + else + { + rate_limit_io_unit = static_cast(parsed); + } } if (reader.HasValue(sec_run, "rate_bg_ratio")) { diff --git a/src/storage/shard.cpp b/src/storage/shard.cpp index 1faa3ee04..65db7f62e 100644 --- a/src/storage/shard.cpp +++ b/src/storage/shard.cpp @@ -139,14 +139,13 @@ void Shard::CollectPeriodicGauges(metrics::Meter *meter) meter->Collect(metrics::NAME_ELOQSTORE_LOCAL_SPACE_USED, static_cast(io_mgr_->GetLocalSpaceUsed())); - // The M1/M2 count budgets (and their per-class in-flight gauges) are - // retired in favor of the M4 rate budget (docs/design/io_qos.md). The - // closest surviving instantaneous-depth gauge is the class-blind - // in-flight device-command window; report it under the read-pages - // metric name until dedicated rate-budget metrics are defined. - const IoQosStats qos = io_mgr_->GetIoQosStats(); - meter->Collect(metrics::NAME_ELOQSTORE_INFLIGHT_READ_PAGES, - static_cast(qos.io_window_inflight_)); + // The per-class in-flight page gauges belonged to the retired M1/M2 + // count budgets; the M4 rate budget exposes no equivalent per-class + // instantaneous depth, so nothing is collected for them here (they are + // no longer registered). Reporting the class-blind command window under + // a "read pages" name would mislabel writes as reads and read zero + // whenever the window is disabled, so it is deliberately not done. + // Dedicated rate-budget metrics are a follow-up (docs/design/io_qos.md). } #endif diff --git a/tests/io_qos.cpp b/tests/io_qos.cpp index 0152cb564..98e46dba8 100644 --- a/tests/io_qos.cpp +++ b/tests/io_qos.cpp @@ -116,23 +116,28 @@ TEST_CASE("rate budget: charges device IO and completes under a tiny rate", TEST_CASE("rate budget: foreground borrows background's idle surplus", "[io_qos]") { - // Reads run after all writes have completed, so the background class - // is idle and its share should be lent to foreground: with a rate low - // enough that foreground exhausts its own 75% share, some read - // admissions must be granted from the background bucket (borrowed), - // and every op must still be accounted to the borrower's class. + // Reads run after all writes complete, so the background class is idle + // and its share must be lent to foreground. A single 600KB overflow + // value read issues ~150 page reads concurrently (one ReadPages batch), + // so foreground demand instantaneously exceeds its whole burst of ops + // and the surplus must be granted from the idle background bucket — + // deterministic regardless of device speed (a rate-based sequential + // read stream is not: it can stay under the foreground share). Every + // borrowed op is still accounted to the foreground (borrower) class. eloqstore::KvOptions opts = default_opts; opts.disk_rate_limit_iops = 2000; opts.rate_limit_burst_ms = 4; + opts.overflow_pointers = 128; eloqstore::EloqStore *store = InitStore(opts); MapVerifier verify(test_tbl_id, store, false); - verify.SetValueSize(200); - verify.WriteRnd(0, 500, 0, 25); - for (int i = 0; i < 100; i++) - { - verify.Read(std::rand() % 500); - } + verify.SetValueSize(600 * 1024); + verify.Upsert(1); + verify.Upsert(2); + // Background is idle now; these foreground reads each fan out into a + // large concurrent page-read batch that outruns the foreground burst. + verify.Read(1); + verify.Read(2); eloqstore::IoQosStats stats = ShardStats(store); REQUIRE(stats.rate_.admitted_ops_ > 0); @@ -365,13 +370,31 @@ TEST_CASE("io qos stats: concurrent sampling", "[io_qos][stats]") REQUIRE(stats.io_window_inflight_ == 0); } +TEST_CASE("rate budget: write ops cost is ceil(bytes / io_unit)", "[io_qos]") +{ + // The write ops charge is the single WriteRateOps formula used by both + // WritePage and SubmitMergedWrite. At the 4KB default quantum a data + // page costs 1 op and a 1MB merged write costs 256; a finer 2KB unit + // doubles both. Locks the currency so the SubmitMergedWrite clamp + // regression (which floored the unit at data_page_size) cannot return. + using eloqstore::IouringMgr; + REQUIRE(IouringMgr::WriteRateOpsFor(4 * 1024, 4 * 1024) == 1); + REQUIRE(IouringMgr::WriteRateOpsFor(1u << 20, 4 * 1024) == 256); + REQUIRE(IouringMgr::WriteRateOpsFor(4 * 1024, 2 * 1024) == 2); + REQUIRE(IouringMgr::WriteRateOpsFor(1u << 20, 2 * 1024) == 512); + // Partial final unit rounds up. + REQUIRE(IouringMgr::WriteRateOpsFor(4 * 1024 + 1, 4 * 1024) == 2); +} + TEST_CASE("rate budget: bytes bucket alone paces merged writes", "[io_qos]") { - // Only the bytes bucket enabled (iops = 0): append-mode merged writes - // must charge bytes and complete. Exercises the ops-disabled branch of - // Positive() and the large-cost debt path (a merged write can exceed - // one burst of byte tokens). + // Only the bytes bucket enabled: append_opts inherits the nonzero + // default disk_rate_limit_iops, so it must be explicitly zeroed or this + // would enable both buckets and never exercise the ops-disabled branch + // of Positive() / the ops-not-debited path in Charge(). The tiny byte + // budget must actually block background writes. eloqstore::KvOptions opts = append_opts; + opts.disk_rate_limit_iops = 0; opts.disk_rate_limit_mbps = 8; opts.rate_limit_burst_ms = 4; eloqstore::EloqStore *store = InitStore(opts); @@ -385,6 +408,10 @@ TEST_CASE("rate budget: bytes bucket alone paces merged writes", "[io_qos]") } eloqstore::IoQosStats stats = ShardStats(store); - REQUIRE(stats.rate_.admitted_bytes_ > 0); REQUIRE(stats.bg_rate_.admitted_bytes_ > 0); + // The small byte budget must have paced (blocked) background writes. + // (Ops are still counted in admitted_ops_ for observability even with + // the ops dimension disabled; only the ops *balance* is left untouched, + // which is what the overflow fix guards — not asserted here.) + REQUIRE(stats.bg_rate_.blocked_count_ > 0); } From 7376b7e07f756b05d5f6210c0bf5477050a617b0 Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Wed, 5 Aug 2026 13:55:46 -0700 Subject: [PATCH 26/30] fix(shard): admit new requests through ready_tasks_, not inline at intake Perf testing uncovered a scheduling fairness gap: new KV requests and resumed tasks flowed through two different paths. StartTask ran each dequeued request's first coroutine segment inline during intake (up to 128 per loop iteration, unconditionally), while tasks resumed by IO completions or rate-budget grants waited in ready_tasks_ for the scheduler's bounded window (max_processing_time_microseconds). Under sustained load the loop therefore preferred starting new work over finishing in-flight work; the ready queue backlog grew and mid-flight tasks - including peek-and-grant wakes already holding charged rate tokens - were progressively deferred, inflating tail latency. StartTask now creates the coroutine SUSPENDED: the body hands control straight back to the creator (the same continuation handoff Yield uses) before touching the request lambda, and the task is enqueued into ready_tasks_. ExecuteReadyTasks becomes the single scheduling point, so new and resumed work runs in true arrival order under one time budget. The request lambda stays captured in the coroutine frame - no type erasure, no KvTask API change; a ready task is uniformly "just resume". Consequences kept intact by construction: - ProcessReq's false-return contract is unchanged (task-acquisition failures happen before StartTask); per-table write serialization and the reopen paths are untouched. - The idle wait cannot false-trigger: a created task counts in TaskManager::NumActive(). - The TXSERVICE request-latency clock is captured at creation (it used to be read inside the body, which was the same instant only because creation and first execution coincided), so measured durations still include the ready-queue wait. - cur_resume_start_us_ stamping moves to ExecuteReadyTasks' resume, which already does it; the two-instruction prologue needs no mark. Cost: one extra continuation switch per request and one loop iteration of added first-segment latency, in exchange for arrival-order fairness under load. Full suite green (327/327). Tail-latency validation on the perf VM pending (offline). Co-Authored-By: Claude Fable 5 --- include/storage/shard.h | 53 +++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/include/storage/shard.h b/include/storage/shard.h index 249f1cbaa..45ab31a93 100644 --- a/include/storage/shard.h +++ b/include/storage/shard.h @@ -154,30 +154,53 @@ class Shard // module worker (WorkOneRound), the sole consumer of requests_. void DrainPendingRequests(); + /** + * @brief Create a task's coroutine SUSPENDED and admit it through + * ready_tasks_ instead of executing its first segment inline. + * + * The coroutine body yields straight back to the creator before running + * the request lambda, so callcc only pays the creation prologue here; + * the scheduler's ordinary resume in ExecuteReadyTasks runs the first + * segment. This makes ready_tasks_ the single scheduling point for new + * and in-flight work: previously each dequeued request executed its + * first segment inline during intake (up to 128 per loop iteration), + * ahead of tasks already resumed by IO completions or budget grants, + * which gradually starved mid-flight tasks under high load and inflated + * tail latency. + */ template void StartTask(KvTask *task, KvRequest *req, F lbd) { task->req_ = req; task->result_err_ = KvError::NoError; task->status_ = TaskStatus::Ongoing; +#ifdef ELOQSTORE_WITH_TXSERVICE + // Captured at creation so the measured request duration includes the + // ready-queue wait (before the suspended-creation change, creation + // and first execution were the same instant). + metrics::TimePoint request_start{}; + if (this->store_->EnableMetrics()) + { + request_start = metrics::Clock::now(); + } +#endif running_ = task; - // Mark the resume start so a cooperative background loop measures this - // first segment from now, not from the previously resumed task's - // timestamp (see CurResumeElapsedUs / MaybeYield). - cur_resume_start_us_ = ReadTimeMicroseconds(); task->coro_ = boost::context::callcc( std::allocator_arg, stack_allocator_, - [lbd, this](continuation &&sink) - { + [lbd, + this #ifdef ELOQSTORE_WITH_TXSERVICE - metrics::TimePoint request_start; - if (this->store_->EnableMetrics()) - { - request_start = metrics::Clock::now(); - } + , + request_start #endif + ](continuation &&sink) + { shard->main_ = std::move(sink); + // Created suspended: hand control straight back to StartTask + // without running the request body; the scheduler's first + // resume() continues from here. + shard->main_ = shard->main_.resume(); KvError err = lbd(); KvTask *task = ThdTask(); task->result_err_ = err; @@ -195,10 +218,10 @@ class Shard return std::move(shard->main_); }); running_ = nullptr; - if (task->status_ == TaskStatus::Finished) - { - OnTaskFinished(task); - } + // The prologue cannot finish the task; it is admitted like any + // resumed task, in arrival order relative to them. + assert(task->status_ == TaskStatus::Ongoing); + ready_tasks_.Enqueue(task); } moodycamel::BlockingConcurrentQueue requests_; From 64110a623769868790d7f1141fd659d0c074a2c2 Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Thu, 6 Aug 2026 00:50:33 -0700 Subject: [PATCH 27/30] perf(shard): issue prepared IO in the same round it is prepared The shard loop ran ExecuteReadyTasks() last, so SQEs it prepared were not handed to the kernel until the NEXT round's Submit(). In module mode (the embedding runtime drives Process/HasTask) the next round is an external scheduling decision, so every IO hop paid a full scheduling quantum of dead device time. The fairness fix (7376b7e) made this worse for new requests: StartTask now only enqueues, so a request crossed two round boundaries before its first IO reached the device instead of one. Reorder every round implementation to Submit -> PollComplete -> Promote -> intake -> ExecuteReadyTasks -> Flush so all four producers into ready_tasks_ (rate-budget grants from Submit's RefillAndWake, IO completions, delayed reopens, new requests) land before the single ExecuteReadyTasks, and the SQEs it prepares are issued before the thread is handed back. WorkOneRound already dequeued first, so module mode needed only the trailing flush. Submit() keeps the top-of-round slot deliberately: it owns RefillAndWake (the rate budget's only wake source) and the kernel entry that delivers CQEs, which PollComplete cannot do itself under IORING_SETUP_DEFER_TASKRUN. So the flush is a separate FlushSubmit() rather than a second Submit(), keeping the refill at exactly one per round; it no-ops when the round prepared nothing and leaves consecutive_skipped_submits_ untouched so the DEFER_TASKRUN forced-enter safety net stays owned by Submit. Measured (Azure L-series, 8 partitions / 4 shards, QD128 storm, 4 interleaved rounds per arm): a wash, as expected for standalone mode where rounds are microseconds apart and foreground is pinned to the rate-budget cap. Median 205,851 vs 205,853 QPS, p99 2285 vs 2281 us, p99.9 2636 vs 2590 us (ranges overlap), writes 73 MB/s in every run. QD32 likewise. The extra kernel entry did not materialize: io_uring_enter over matched 50s runs fell 5.7% (106.0M vs 112.4M), because Submit's no-op path now usually finds nothing prepared while intake-before-execute batches more SQEs behind one flush. Co-Authored-By: Claude Fable 5 --- docs/architecture/04-execution-model.md | 30 +++++++++++++++++++------ include/async_io_manager.h | 20 +++++++++++++++++ src/async_io_manager.cpp | 23 +++++++++++++++++++ src/storage/shard.cpp | 21 ++++++++++++----- 4 files changed, 81 insertions(+), 13 deletions(-) diff --git a/docs/architecture/04-execution-model.md b/docs/architecture/04-execution-model.md index d1d959ca2..682e6803b 100644 --- a/docs/architecture/04-execution-model.md +++ b/docs/architecture/04-execution-model.md @@ -23,16 +23,32 @@ shard context without parameter plumbing. `Shard::WorkLoop()` (normal build) repeats: -1. `io_mgr_->Submit()` — flush prepared io_uring SQEs. -2. `io_mgr_->PollComplete()` — reap CQEs; each completion either finishes a - blocked task's I/O (`FinishIo`) or processes a background write request. - Cloud/standby ready-queues are drained here too. -3. `ExecuteReadyTasks()` — resume coroutines from `ready_tasks_`, then (when - the normal queue is empty) `low_priority_ready_tasks_` (background - compaction/GC yield here to protect foreground latency). +1. `io_mgr_->Submit()` — refill the device rate budget (`RefillAndWake`, its + only wake source) and enter the kernel. The ring uses + `IORING_SETUP_DEFER_TASKRUN`, so this entry is also what delivers CQEs. +2. `io_mgr_->PollComplete()` — reap CQEs (pure user space: `peek_cqe` + + `for_each_cqe`); each completion either finishes a blocked task's I/O + (`FinishIo`) or processes a background write request. Cloud/standby + ready-queues are drained here too. +3. `PromoteReadyDelayedReopenRequests()`. 4. Dequeue up to 128 new `KvRequest`s from the MPSC `requests_` queue (blocking with 100 ms timeout only when fully idle) and feed each to `OnReceivedReq`. +5. `ExecuteReadyTasks()` — resume coroutines from `ready_tasks_`, then (when + the normal queue is empty) `low_priority_ready_tasks_` (background + compaction/GC yield here to protect foreground latency). +6. `io_mgr_->FlushSubmit()` — issue the SQEs this round prepared. + +**The order is load-bearing.** Four producers feed `ready_tasks_` — rate-budget +grants (step 1), I/O completions (step 2), delayed reopens (step 3), and new +requests (step 4) — and all of them are placed before the single +`ExecuteReadyTasks`, so work admitted in a round runs in that same round. +`FlushSubmit` then closes the loop on the output side: without it, SQEs +prepared in step 5 would wait for the *next* round's `Submit`, which in module +mode is a full external scheduling quantum of dead device time on every I/O +hop. `FlushSubmit` deliberately does not touch `consecutive_skipped_submits_` +(the DEFER_TASKRUN forced-enter safety net stays owned by `Submit`) and is a +no-op when the round prepared nothing. Exit: when the store is stopping and the shard is idle. Teardown runs on the shard thread: `TaskManager::Shutdown()` → `PageManager::Shutdown()` → diff --git a/include/async_io_manager.h b/include/async_io_manager.h index 836dede3a..dcb769452 100644 --- a/include/async_io_manager.h +++ b/include/async_io_manager.h @@ -245,8 +245,27 @@ class AsyncIoManager { return store_stopping_.load(std::memory_order_acquire); } + /** + * @brief Top-of-round kernel entry: refill the device rate budget (its + * only wake source) and drive io_uring submission / task_work. + */ virtual void Submit() = 0; virtual void PollComplete() = 0; + /** + * @brief End-of-round flush: push SQEs prepared by ExecuteReadyTasks to + * the kernel in the same round instead of leaving them for the next + * round's Submit. + * + * Splitting this out of Submit keeps the rate-budget refill at exactly + * one per round while removing the round-boundary delay between + * preparing an IO and issuing it — which in module mode (an embedding + * runtime driving Process/HasTask) is a full external scheduling + * quantum of dead device time per IO hop. No-op when the round prepared + * nothing. + */ + virtual void FlushSubmit() + { + } virtual bool NeedPrewarm() const { return false; @@ -698,6 +717,7 @@ class IouringMgr : public AsyncIoManager ~IouringMgr() override; KvError Init(Shard *shard) override; void Submit() override; + void FlushSubmit() override; /** * @brief A shard must not idle-block while this ring has prepared or * submitted-but-unreaped IO: CQE delivery under DEFER_TASKRUN diff --git a/src/async_io_manager.cpp b/src/async_io_manager.cpp index 121ed4ff7..7fddf5421 100644 --- a/src/async_io_manager.cpp +++ b/src/async_io_manager.cpp @@ -2401,6 +2401,29 @@ void IouringMgr::Submit() } } +void IouringMgr::FlushSubmit() +{ + if (prepared_sqe_ == 0) + { + // Nothing was prepared after the round's Submit. Kernel re-entry + // (and with it the DEFER_TASKRUN forced-enter safety net) is + // Submit's job at the top of the next round, so deliberately do + // not touch consecutive_skipped_submits_ here: this call must not + // perturb that cadence. + return; + } + // A round that prepared SQEs is entering the kernel now, so the skip + // counter restarts exactly as it would in Submit. + consecutive_skipped_submits_ = 0; + int ret = io_uring_submit(&ring_); + if (__builtin_expect(ret < 0, 0)) + { + LOG(ERROR) << "iouring flush submit failed " << ret; + return; + } + prepared_sqe_ -= ret; +} + void IouringMgr::PollComplete() { if (io_stats_enabled_) diff --git a/src/storage/shard.cpp b/src/storage/shard.cpp index 65db7f62e..5dd72f794 100644 --- a/src/storage/shard.cpp +++ b/src/storage/shard.cpp @@ -220,7 +220,6 @@ void Shard::WorkLoop() io_mgr_->Submit(); io_mgr_->PollComplete(); PromoteReadyDelayedReopenRequests(); - ExecuteReadyTasks(); int nreqs = dequeue_requests(); if (nreqs < 0) { @@ -230,6 +229,8 @@ void Shard::WorkLoop() { OnReceivedReq(reqs[i]); } + ExecuteReadyTasks(); + io_mgr_->FlushSubmit(); } else { @@ -244,8 +245,6 @@ void Shard::WorkLoop() const uint64_t t2 = ReadTimeMicroseconds(); PromoteReadyDelayedReopenRequests(); const uint64_t t3 = ReadTimeMicroseconds(); - ExecuteReadyTasks(); - const uint64_t t4 = ReadTimeMicroseconds(); uint64_t queue_wait_us = 0; int nreqs = dequeue_requests(&queue_wait_us); if (nreqs < 0) @@ -256,8 +255,12 @@ void Shard::WorkLoop() { OnReceivedReq(reqs[i]); } + const uint64_t t4 = ReadTimeMicroseconds(); + ExecuteReadyTasks(); const uint64_t t5 = ReadTimeMicroseconds(); - const uint64_t total_us = t5 - t0; + io_mgr_->FlushSubmit(); + const uint64_t t6 = ReadTimeMicroseconds(); + const uint64_t total_us = t6 - t0; const uint64_t active_us = total_us - queue_wait_us; if (active_us > 1000) { @@ -269,8 +272,9 @@ void Shard::WorkLoop() LOG(INFO) << "SLOWROUND total=" << total_us << "us active=" << active_us << "us cpu=" << cpu_us << "us submit=" << t1 - t0 << " poll=" << t2 - t1 - << " promote=" << t3 - t2 << " execute=" << t4 - t3 - << " intake=" << t5 - t4 - queue_wait_us + << " promote=" << t3 - t2 + << " intake=" << t4 - t3 - queue_wait_us + << " execute=" << t5 - t4 << " flush=" << t6 - t5 << " queue_wait=" << queue_wait_us << " nreqs=" << nreqs; } @@ -1411,6 +1415,11 @@ void Shard::WorkOneRound() { ExecuteReadyTasks(); } + // Issue what this round prepared before handing the thread back to the + // embedding runtime: the next round is an external scheduling decision + // and may be far away, so leaving SQEs for it would idle the device for + // a whole quantum on every IO hop. + io_mgr_->FlushSubmit(); #ifdef ELOQSTORE_WITH_TXSERVICE // Metrics collection: end of round if (store_->EnableMetrics() && !is_idle_round) From 734ded9c776a6c2d6079fd10dbbe571b3d0367fd Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Thu, 6 Aug 2026 01:27:42 -0700 Subject: [PATCH 28/30] fix(shard): admit new requests after PollComplete in module mode too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WorkOneRound called OnReceivedReq before Submit/PollComplete, so within a round new arrivals were enqueued on ready_tasks_ ahead of the IO completions PollComplete pushes — the precedence the fairness fix (7376b7e) removed from WorkLoop but which module mode still had. Move admission to step 4, matching WorkLoop exactly: Submit -> PollComplete -> Promote -> OnReceivedReq -> Execute -> Flush The dequeue itself stays above because is_idle_round depends on nreqs; only admission (and its req_queue_size_ accounting) moves. Over-reporting req_queue_size_ in the window between the two is safe: only Shard::IsIdle reads it, from the runtime thread via HasTask, where over-reporting is the conservative direction. Compile-verified with -DELOQ_MODULE_ENABLED=ON. WorkOneRound sits inside that ifdef, so the default build (and the 327-test suite, which has no module coverage) never compiles this path. Co-Authored-By: Claude Fable 5 --- docs/architecture/04-execution-model.md | 10 ++++++++-- src/storage/shard.cpp | 15 +++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/docs/architecture/04-execution-model.md b/docs/architecture/04-execution-model.md index 682e6803b..589920a73 100644 --- a/docs/architecture/04-execution-model.md +++ b/docs/architecture/04-execution-model.md @@ -55,8 +55,14 @@ shard thread: `TaskManager::Shutdown()` → `PageManager::Shutdown()` → `io_mgr_->Stop()` (tasks may hold page pins, so task state dies first). In the module build (`ELOQ_MODULE_ENABLED`, doc 10) the same logic is exposed -as `WorkOneRound()` and an external runtime drives it; `IsIdle()` tells the -runtime whether the shard needs another round. +as `WorkOneRound()` (driven by `EloqStoreModule::Process`) and an external +runtime drives it; `IsIdle()` tells the runtime whether the shard needs +another round. It runs the **same step order**, which matters most there: the +gap between rounds is an external scheduling decision rather than a few +microseconds, so both a delayed flush and mis-ordered admission cost a full +quantum. The only structural difference is that the request *dequeue* happens +first because the idle-round test depends on its count; admission +(`OnReceivedReq`) still runs at step 4, after `PollComplete`. ## Tasks are pooled coroutines diff --git a/src/storage/shard.cpp b/src/storage/shard.cpp index 5dd72f794..2dcae50ea 100644 --- a/src/storage/shard.cpp +++ b/src/storage/shard.cpp @@ -1400,6 +1400,17 @@ void Shard::WorkOneRound() #endif } + io_mgr_->Submit(); + + io_mgr_->PollComplete(); + PromoteReadyDelayedReopenRequests(); + + // Admit new requests only after PollComplete and Promote, matching + // WorkLoop: everything already in flight is enqueued on ready_tasks_ + // ahead of this round's arrivals, so a burst of new requests cannot + // take precedence over tasks the shard has already started. The + // dequeue itself stays above (is_idle_round depends on nreqs); only + // admission moves. for (size_t i = 0; i < nreqs; ++i) { OnReceivedReq(reqs[i]); @@ -1407,10 +1418,6 @@ void Shard::WorkOneRound() req_queue_size_.fetch_sub(nreqs, std::memory_order_relaxed); - io_mgr_->Submit(); - - io_mgr_->PollComplete(); - PromoteReadyDelayedReopenRequests(); if (DurationMicroseconds(ts_) < FLAGS_max_processing_time_microseconds) { ExecuteReadyTasks(); From 834582d7e6c8c3fe6d0c6f87cd8fc86577ae1c08 Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Mon, 17 Aug 2026 00:23:49 -0700 Subject: [PATCH 29/30] feat(module): declare EloqStoreModule's module type brpc's module registry is now keyed by ModuleType rather than by registration order, so every EloqModule must name what it is. Slot kEloqStore is then reserved for this module whether or not it is registered, and --module_visit_order can refer to it as "eloqstore". Co-Authored-By: Claude Opus 5 (1M context) --- include/eloqstore_module.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/eloqstore_module.h b/include/eloqstore_module.h index dad8bef2f..ad7fa095a 100644 --- a/include/eloqstore_module.h +++ b/include/eloqstore_module.h @@ -19,6 +19,11 @@ class EloqStoreModule : public eloq::EloqModule } ~EloqStoreModule() = default; + eloq::ModuleType Type() const override + { + return eloq::ModuleType::kEloqStore; + } + void ExtThdStart(int thd_id) override; void ExtThdEnd(int thd_id) override; void Process(int thd_id) override; From 155255b3c9143909d789bef3c195bda6b50740e3 Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Fri, 21 Aug 2026 23:23:15 -0700 Subject: [PATCH 30/30] fix(io): floor rate_limit_io_unit at 4KB, admit fsyncs through the IO window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address two review findings on #497: - rate_limit_io_unit: ValidateOptions now rejects values below 4KB. WriteRateOps divides by this unit on every write — even with the rate limiter disabled — so zero was a SIGFPE for programmatically constructed KvOptions, which never pass through the INI loader's check. The INI path additionally range-checks the parsed uint64 before narrowing ("4GB" is nonzero but truncates to uint32_t(0)). A sub-page quantum would also overcharge writes, so the floor is the 4KB default rather than just nonzero. - FdatasyncFiles bypassed max_inflight_io: fsync SQEs were tagged BaseReq, so a checkpoint could enqueue fds.size() flush commands past the configured device-queue bound. Batch fsyncs now carry a dedicated BaseReqFsync user-data type, acquire one window command per SQE (a flush occupies a queue slot like any command), and release it per CQE in PollComplete — unconditionally, so failed syncs release too. The rate-budget charge stays 1 op / 0 bytes. The single-fd directory Fdatasync path stays window-exempt with the other metadata ops: it waits per call and cannot burst. Regression tests cover programmatic and INI validation and the fsync error-path acquire/release symmetry; docs updated to match. Co-Authored-By: Claude Fable 5 --- docs/architecture/02-runtime-and-lifecycle.md | 2 +- docs/architecture/07-io-stack.md | 7 +++- docs/design/io_qos.md | 11 +++--- include/async_io_manager.h | 19 +++++++--- include/kv_options.h | 14 +++---- src/async_io_manager.cpp | 16 +++++++- src/eloq_store.cpp | 8 ++++ src/kv_options.cpp | 8 +++- tests/eloq_store_test.cpp | 38 +++++++++++++++++++ tests/io_qos.cpp | 36 ++++++++++++++++++ 10 files changed, 135 insertions(+), 24 deletions(-) diff --git a/docs/architecture/02-runtime-and-lifecycle.md b/docs/architecture/02-runtime-and-lifecycle.md index 192153717..0a1dd35cf 100644 --- a/docs/architecture/02-runtime-and-lifecycle.md +++ b/docs/architecture/02-runtime-and-lifecycle.md @@ -114,7 +114,7 @@ state: /`disk_rate_limit_mbps` (per-disk device rate limit, on by default, divided across shards), `rate_bg_ratio` (background share of the rate — the tail-predictability policy knob), `rate_limit_burst_ms`, - `rate_limit_io_unit` (write ops quantum), and `max_inflight_io` (optional + `rate_limit_io_unit` (write ops quantum, minimum 4KB), and `max_inflight_io` (optional class-blind in-flight command window). The former count budgets `max_inflight_read`/`bg_read_ratio` are deprecated no-ops. - **Cloud**: `cloud_provider/endpoint/region/keys`, `local_space_limit` + diff --git a/docs/architecture/07-io-stack.md b/docs/architecture/07-io-stack.md index 9e8a8aed1..543e3959b 100644 --- a/docs/architecture/07-io-stack.md +++ b/docs/architecture/07-io-stack.md @@ -56,8 +56,11 @@ Responsibilities: (`disk_rate_limit_iops`/`disk_rate_limit_mbps`, on by default) are divided across shards (`× store paths / num_threads`). A separate optional class-blind in-flight command window (`max_inflight_io`, off by default) is - the only occupancy-style cap and *does* release per CQE in `PollComplete`. - Metadata, manifest, bulk file/snapshot paths, and segment IO are exempt. + the only occupancy-style cap and *does* release per CQE in `PollComplete`; + it charges 1 per page IO, `ceil(len / 256KB)` per merged write, and 1 per + batch fsync (`FdatasyncFiles`, tagged `BaseReqFsync` so the release can + tell them from exempt metadata ops). Other metadata, manifest, bulk + file/snapshot paths, and segment IO are exempt from both mechanisms. `GetIoQosStats()` (also `EloqStore::GetIoQosStats(shard_id)`) exposes the per-class rate counters (blocked count/us, admitted ops/bytes, borrowed ops) and the window's in-flight/high-water/blocked gauges. diff --git a/docs/design/io_qos.md b/docs/design/io_qos.md index c9228cea8..59847a9c0 100644 --- a/docs/design/io_qos.md +++ b/docs/design/io_qos.md @@ -421,11 +421,12 @@ uint32_t rate_limit_io_unit = 4096; // write ops quantum: ceil(len/unit) // ops per write (WriteRateOps, used by // both WritePage and SubmitMergedWrite). // Default = one data page: 4KB write = - // 1 op, 1MB merged = 256. Must be - // nonzero (validated at load). A finer - // unit paces background harder; not - // needed at the tested read-tail - // target. + // 1 op, 1MB merged = 256. Minimum 4KB + // (= default; ValidateOptions rejects + // smaller, the INI loader keeps the + // default on malformed/out-of-range + // values). Coarsen only if the device + // accounts in larger units. uint32_t rate_bg_ratio = 25; // background share of the rate, percent ``` diff --git a/include/async_io_manager.h b/include/async_io_manager.h index dcb769452..1ce7e8ffd 100644 --- a/include/async_io_manager.h +++ b/include/async_io_manager.h @@ -964,8 +964,13 @@ class IouringMgr : public AsyncIoManager // KvTask / BaseReq types, plus a read-budget release. The distinct // types exist so PollComplete can tell budgeted data-page reads // apart from metadata ops that share the plain payload types. - KvTaskPageRead, // ReadPage (single page), payload = KvTask* - BaseReqPageRead // ReadPages (one page of a batch), payload = BaseReq* + KvTaskPageRead, // ReadPage (single page), payload = KvTask* + BaseReqPageRead, // ReadPages (one page of a batch), payload = BaseReq* + // Batch fsyncs charged against the io window (FdatasyncFiles), + // payload = BaseReq*. Distinct from BaseReq so PollComplete can + // release the window command without touching the metadata ops + // (open, close, unlink, ...) that stay window-exempt. + BaseReqFsync }; struct BaseReq @@ -1396,8 +1401,9 @@ class IouringMgr : public AsyncIoManager * rate governs allocation per second with class policy; this bounds * the instantaneous outstanding window toward the device, smoothing * the rate bucket's burst release. Cost is device commands: 1 per page - * IO, ceil(len / kDeviceCmdBytes) per merged write. Oversized-request - * escape: a cost above the cap admits alone once the + * IO, ceil(len / kDeviceCmdBytes) per merged write, 1 per batch fsync + * (FdatasyncFiles — a flush occupies a queue slot like any command). + * Oversized-request escape: a cost above the cap admits alone once the * window drains. Completion-driven wake in PollComplete. */ void AcquireIoWindow(uint32_t cost); @@ -1425,8 +1431,9 @@ class IouringMgr : public AsyncIoManager * both WritePage (one data page) and SubmitMergedWrite use it, so a * page's cost and a merged write's cost are consistent by construction. * The unit is the configured physical write quantum (default = data - * page size); it is validated nonzero at option load, so no clamp is - * needed here. + * page size); EloqStore::ValidateOptions rejects units below 4KB — + * covering programmatic KvOptions, not just the INI path — so the + * divisor is never zero and no clamp is needed here. */ static uint32_t WriteRateOpsFor(size_t bytes, uint32_t unit) { diff --git a/include/kv_options.h b/include/kv_options.h index 3349d984e..cbb7d93d1 100644 --- a/include/kv_options.h +++ b/include/kv_options.h @@ -140,13 +140,13 @@ struct KvOptions * operations (WriteRateOps), so a data-page write and a merged write * are metered on one consistent scale. Defaults to the physical write * quantum, one data page (4KB): a 4KB write costs 1 op, a 1MB merged - * write costs 256. Must be nonzero (a zero/malformed value is rejected - * at load and the default is kept). A finer unit (e.g. 2KB) charges - * writes more ops per byte and paces background harder; the tested - * Azure NVMe read-tail target was met at the 4KB default, so it is not - * enabled by default — recalibrate with the fio boundary method in - * docs/design/io_qos.md only if a device's write-accounting unit is - * measured smaller and write throttling needs it. + * write costs 256. Minimum 4KB (= the default): ValidateOptions + * rejects smaller values — the divisor must never be zero, and a + * sub-page quantum would overcharge writes — and LoadFromIni keeps + * the default on a malformed or out-of-uint32-range entry. The tested + * Azure NVMe read-tail target was met at the 4KB default; coarsen + * (recalibrating with the fio boundary method in docs/design/io_qos.md) + * only if a device's write-accounting unit is measured larger. */ uint32_t rate_limit_io_unit = 4 * KB; /** diff --git a/src/async_io_manager.cpp b/src/async_io_manager.cpp index 7fddf5421..64657c4cb 100644 --- a/src/async_io_manager.cpp +++ b/src/async_io_manager.cpp @@ -2458,6 +2458,7 @@ void IouringMgr::PollComplete() task->io_flags_ = cqe->flags; break; case UserDataType::BaseReqPageRead: + case UserDataType::BaseReqFsync: case UserDataType::BaseReq: { BaseReq *req = static_cast(ptr); @@ -2466,6 +2467,13 @@ void IouringMgr::PollComplete() TEST_FAIL_POINT_ACTION("BaseReqPageReadCqe", cqe->res = -EIO); ReleaseIoWindow(1); } + else if (type == UserDataType::BaseReqFsync) + { + TEST_FAIL_POINT_ACTION("BaseReqFsyncCqe", cqe->res = -EIO); + // Mirrors the per-SQE acquire in FdatasyncFiles; + // unconditional so failed CQEs release their command too. + ReleaseIoWindow(1); + } req->res_ = cqe->res; req->flags_ = cqe->flags; task = req->task_; @@ -2802,7 +2810,13 @@ KvError IouringMgr::FdatasyncFiles(const TableIdent &tbl_id, // enough space for this vector so that it will never reallocate. const FsyncReq &req = reqs.emplace_back(ThdTask(), fd_ref); auto [fd, registered] = req.fd_ref_.FdPair(); - io_uring_sqe *sqe = GetSQE(UserDataType::BaseReq, &req); + // Window admission (M2): a flush occupies a device queue slot like + // any command, so a checkpoint's batch must not bypass the cap. + // Per-SQE acquire, like the page-read paths; a mid-loop wait lets + // the shard loop submit the already-charged SQEs. Released once + // per BaseReqFsync CQE in PollComplete. + AcquireIoWindow(1); + io_uring_sqe *sqe = GetSQE(UserDataType::BaseReqFsync, &req); if (registered) { sqe->flags |= IOSQE_FIXED_FILE; diff --git a/src/eloq_store.cpp b/src/eloq_store.cpp index 0509b627e..4fa15f1ce 100644 --- a/src/eloq_store.cpp +++ b/src/eloq_store.cpp @@ -148,6 +148,14 @@ bool EloqStore::ValidateOptions(KvOptions &opts) LOG(ERROR) << "Option max_global_request_batch cannot be zero"; return false; } + // WriteRateOps divides by this on every write, even with the rate + // limiter disabled; a sub-page quantum would also overcharge writes. + if (opts.rate_limit_io_unit < 4 * KB) + { + LOG(ERROR) << "Option rate_limit_io_unit (" << opts.rate_limit_io_unit + << ") must be at least 4KB"; + return false; + } if ((opts.data_page_size & (page_align - 1)) != 0) { LOG(ERROR) << "Option data_page_size is not page aligned"; diff --git a/src/kv_options.cpp b/src/kv_options.cpp index 6e52d80ee..256b6fd2a 100644 --- a/src/kv_options.cpp +++ b/src/kv_options.cpp @@ -189,10 +189,14 @@ int KvOptions::LoadFromIni(const char *path) { std::string io_unit_str = reader.Get(sec_run, "rate_limit_io_unit", ""); const uint64_t parsed = ParseSizeWithUnit(io_unit_str); - if (parsed == 0) + // Range check on the wide type before narrowing: e.g. "4GB" is + // nonzero as uint64_t but truncates to uint32_t(0), which would + // reach WriteRateOps' division. Minimum is 4KB (see kv_options.h). + if (parsed < static_cast(4 * KB) || + parsed > std::numeric_limits::max()) { LOG(WARNING) << "rate_limit_io_unit '" << io_unit_str - << "' is invalid; keeping default " + << "' is invalid (minimum 4KB); keeping default " << rate_limit_io_unit << " bytes"; } else diff --git a/tests/eloq_store_test.cpp b/tests/eloq_store_test.cpp index cf649dd09..0a9a199d7 100644 --- a/tests/eloq_store_test.cpp +++ b/tests/eloq_store_test.cpp @@ -77,6 +77,32 @@ TEST_CASE("KvOptions parses QoS knobs and preserves malformed defaults", REQUIRE(options.LoadFromIni(ini_path.c_str()) == 0); REQUIRE(options.max_inflight_write == eloqstore::KvOptions{}.max_inflight_write); + + // rate_limit_io_unit: a valid size within [4KB, UINT32_MAX] is taken; + // below the 4KB minimum or above uint32 range ("4GB" is nonzero as + // uint64_t but would truncate to 0) keeps the default. + { + std::ofstream ini(ini_path); + REQUIRE(ini.is_open()); + ini << "[run]\nrate_limit_io_unit = 8KB\n" + "[permanent]\nstore_path = /tmp/unused\n"; + } + options = eloqstore::KvOptions{}; + REQUIRE(options.LoadFromIni(ini_path.c_str()) == 0); + REQUIRE(options.rate_limit_io_unit == 8 * 1024); + for (const char *bad : {"2KB", "4GB", "0"}) + { + { + std::ofstream ini(ini_path); + REQUIRE(ini.is_open()); + ini << "[run]\nrate_limit_io_unit = " << bad + << "\n[permanent]\nstore_path = /tmp/unused\n"; + } + options = eloqstore::KvOptions{}; + REQUIRE(options.LoadFromIni(ini_path.c_str()) == 0); + REQUIRE(options.rate_limit_io_unit == + eloqstore::KvOptions{}.rate_limit_io_unit); + } CleanupTestDir(test_dir); } @@ -93,6 +119,18 @@ TEST_CASE("EloqStore ValidateOptions validates all parameters", "[eloq_store]") REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == false); options = CreateValidOptions(test_dir); // restore valid value + // rate_limit_io_unit floors at 4KB. WriteRateOps divides by it on + // every write — even with the rate limiter disabled — so zero and + // sub-page quanta must be rejected for programmatically constructed + // options, not just on the INI path. + options.rate_limit_io_unit = 0; + REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == false); + options.rate_limit_io_unit = 2 * 1024; + REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == false); + options.rate_limit_io_unit = 8 * 1024; + REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == true); + options = CreateValidOptions(test_dir); // restore valid value + // Test data_page_size that is not page-aligned options.data_page_size = 4097; // not page-aligned REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == false); diff --git a/tests/io_qos.cpp b/tests/io_qos.cpp index 98e46dba8..8cdd89ab9 100644 --- a/tests/io_qos.cpp +++ b/tests/io_qos.cpp @@ -319,6 +319,42 @@ TEST_CASE("io window: negative MergedWriteReq CQE releases and recovers", REQUIRE(ShardStats(store).io_window_inflight_ == 0); } +TEST_CASE("io window: negative fsync CQE releases and recovers", "[io_qos]") +{ + // FdatasyncFiles charges one window command per fsync SQE + // (BaseReqFsync) so a checkpoint's flush batch cannot bypass + // max_inflight_io. Acquire and release must stay symmetric on the + // error path: the failed CQE still releases its command (in Debug the + // ReleaseIoWindow underflow assert would fire if the acquire were + // missing), the failure surfaces on the write request, and the store + // keeps serving afterwards. + eloqstore::KvOptions opts = default_opts; + opts.disk_rate_limit_iops = 0; + opts.max_inflight_io = 2; + eloqstore::EloqStore *store = InitStore(opts); + const eloqstore::TableIdent tbl_id{"qos-fsync-cqe", 0}; + const uint64_t ts = utils::UnixTs(); + + eloqstore::BatchWriteRequest failed; + failed.SetTableId(tbl_id); + failed.AddWrite("fsync-key", "value", ts, eloqstore::WriteOp::Upsert); + eloqstore::FailPoint::GetInstance().ArmOnce("BaseReqFsyncCqe"); + store->ExecSync(&failed); + eloqstore::FailPoint::GetInstance().Disarm(); + + REQUIRE(failed.Error() == eloqstore::KvError::IoFail); + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.fdatasync_count_ > 0); + REQUIRE(stats.io_window_inflight_ == 0); + + eloqstore::BatchWriteRequest recovery; + recovery.SetTableId(tbl_id); + recovery.AddWrite("recovery", "value", ts + 1, eloqstore::WriteOp::Upsert); + store->ExecSync(&recovery); + REQUIRE(recovery.Error() == eloqstore::KvError::NoError); + REQUIRE(ShardStats(store).io_window_inflight_ == 0); +} + TEST_CASE("io qos stats: concurrent sampling", "[io_qos][stats]") { // IoQosStats must be safely sampleable from another thread while the