diff --git a/CLAUDE.md b/CLAUDE.md index e9610510..dc505300 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,6 +71,8 @@ Full subsystem documentation lives in `docs/architecture/` (see above). The load **I/O stack is layered, not parallel implementations** (`include/async_io_manager.h`): `IouringMgr` is the local-filesystem base (page buffers, manifests, FD cache, direct I/O, merged writes); `CloudStoreMgr` extends it, treating local files as a cache over object storage (`ObjectStore` → `CloudStorageService` → `AsyncHttpManager`); `StandbyStoreMgr` extends it filling missing state via rsync jobs in `StandbyService`. Local manifest/page semantics are the common substrate in every mode — remote sync and cache cleanup must preserve local replayability. +**Page-I/O budget symmetry:** acquire after every other blocking resource, do not yield between admission and SQE submission, and release the identical cost exactly once per CQE. + **On-disk and in-memory page formats intentionally differ** (`src/storage/`). `DataPageBuilder`/`IndexPageBuilder` define the canonical encoded layout (prefix-compressed keys, restart points, timestamp deltas, overflow flags); large values go to chained overflow pages. In memory, internal nodes are swizzled `MemIndexPage` objects with pins and cached child pointers, managed by `IndexPageManager`. Any file-format change must update both builder and reader paths. **Durability is anchored at RootMeta + manifest, not pages** (`src/storage/root_meta.cpp`, `src/storage/page_mapper.cpp`, `src/replayer.cpp`). Logical-to-file page identity is versioned through `MappingSnapshot`s for COW root transitions; `PageMapper` owns the logical page table and allocator, reconstructed from manifest replay. `Replayer` handles both cold restart and live refresh (reopen, cloud/standby snapshot install) — manifest encoding changes must keep both working. Swizzled pages can only be recycled after being unswizzled from every live mapping snapshot. diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 6ad4d8e5..6cf29b87 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -62,6 +62,10 @@ target_link_libraries(load_bench eloqstore ${BRPC_LIB} ${GFLAGS_LIBRARY}) add_executable(simple_bench simple_bench.cpp) target_link_libraries(simple_bench eloqstore ${BRPC_LIB} ${GFLAGS_LIBRARY}) +# IO QoS interference benchmark (docs/design/io_qos.md, plan commit 3). +add_executable(interference_bench interference_bench.cpp) +target_link_libraries(interference_bench eloqstore ${GFLAGS_LIBRARY}) + add_executable(simple_test simple_test.cpp) target_link_libraries(simple_test eloqstore ${BRPC_LIB} ${GFLAGS_LIBRARY}) diff --git a/benchmark/eloq_store_bm.cc b/benchmark/eloq_store_bm.cc index 2c066dc0..99f0397d 100644 --- a/benchmark/eloq_store_bm.cc +++ b/benchmark/eloq_store_bm.cc @@ -1,11 +1,24 @@ #include "eloq_store_bm.h" +#include + +#include +#include #include +#include #include #include +#include +// https://github.com/cameron314/concurrentqueue/issues/280 +#undef BLOCK_SIZE +#include "../external/concurrentqueue/blockingconcurrentqueue.h" #include "kv_options.h" +DECLARE_uint32(client_threads); +DECLARE_uint32(inflight_per_client); +DECLARE_uint32(per_shard_cap); + namespace EloqStoreBM { static const std::string table_name_str = "eloq_store_bm"; @@ -530,6 +543,250 @@ void Benchmark::GenBatchRecord(const Benchmark &bm, #endif } +namespace +{ +struct Get2Client +{ + moodycamel::BlockingConcurrentQueue done_; + std::vector lat_us_; + uint64_t outstanding_{0}; + uint64_t read_failed_{0}; + uint64_t issue_failed_{0}; +}; + +uint64_t Get2NowUs() +{ + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} +} // namespace + +void Benchmark::OnReadV2(::eloqstore::KvRequest *req) +{ + auto *op = reinterpret_cast(req->UserData()); + CHECK(static_cast(op->client_)->done_.enqueue(op)) + << "GET2 completion queue allocation failed"; +} + +void Benchmark::RunGet2(uint32_t client_threads, + uint32_t inflight, + uint32_t per_shard_cap) +{ + CHECK_GT(client_threads, 0U) << "GET2 client_threads must be positive"; + CHECK_GT(inflight, 0U) << "GET2 inflight_per_client must be positive"; + CHECK_GT(partition_count_, 0U) << "GET2 partition_count must be positive"; + CHECK_GE(key_maximum_, key_minimum_) + << "GET2 key_maximum must not be less than key_minimum"; + CHECK(per_shard_cap == 0 || + key_maximum_ - key_minimum_ >= partition_count_ - 1); + + const uint16_t nshards = worker_cnt_; + std::vector partition_shards(partition_count_); + std::vector reachable_shards(nshards, false); + for (uint32_t part = 0; part < partition_count_; ++part) + { + const ::eloqstore::TableIdent table_id(table_name_str, part); + partition_shards[part] = table_id.ShardIndex(nshards); + reachable_shards[partition_shards[part]] = true; + } + const uint64_t reachable_count = + std::count(reachable_shards.begin(), reachable_shards.end(), true); + const uint64_t cap_capacity = reachable_count * per_shard_cap; + CHECK(per_shard_cap == 0 || inflight <= cap_capacity) + << "GET2 inflight_per_client=" << inflight + << " exceeds reachable per-shard capacity=" << cap_capacity; + + std::atomic stop{false}; + std::vector clients(client_threads); + std::vector thds; + const uint64_t bench_start = Get2NowUs(); + + for (uint32_t c = 0; c < client_threads; ++c) + { + thds.emplace_back( + [this, + c, + inflight, + per_shard_cap, + nshards, + &partition_shards, + &clients, + &stop]() + { + Get2Client &me = clients[c]; + object_generator gen; + gen.set_random_data(true); + gen.set_random_seed(20260711 + c * 7919); + gen.set_key_size(key_byte_size_); + gen.set_data_size_fixed(value_byte_size_); + gen.set_key_prefix(key_prefix_.data()); + gen.set_key_range(key_minimum_, key_maximum_); + + std::vector ops; + ops.reserve(inflight); + for (uint32_t i = 0; i < inflight; ++i) + { + ops.emplace_back(this); + ops.back().client_ = &me; + } + std::vector shard_out(nshards, 0); + + auto issue = [&](ReadOperation *op) + { + uint64_t key_index = + gen.get_key_index(OBJECT_GENERATOR_KEY_RANDOM); + uint32_t part = key_index % partition_count_; + if (per_shard_cap > 0) + { + uint32_t forward = 0; + for (; forward < partition_count_; ++forward) + { + const uint32_t candidate = + (static_cast(part) + forward) % + partition_count_; + if (shard_out[partition_shards[candidate]] < + per_shard_cap) + { + part = candidate; + break; + } + } + CHECK_LT(forward, partition_count_) + << "GET2 per-shard cap accounting lost capacity"; + if (forward != 0) + { + key_index += forward; + if (key_index > key_maximum_) + { + key_index -= partition_count_; + } + } + } + CHECK_GE(key_index, key_minimum_); + CHECK_LE(key_index, key_maximum_); + CHECK_EQ(key_index % partition_count_, part); + op->shard_ = partition_shards[part]; + op->key_.clear(); + gen.generate_key(key_index, op->key_); + op->req_->SetArgs( + ::eloqstore::TableIdent(table_name_str, part), + op->key_); + op->start_ts_ = Get2NowUs(); + if (!eloq_store_->ExecAsyn(op->req_.get(), + reinterpret_cast(op), + OnReadV2)) + { + ++me.issue_failed_; + return; + } + ++shard_out[op->shard_]; + ++me.outstanding_; + }; + + auto complete = [&](ReadOperation *op) + { + CHECK_GT(me.outstanding_, 0U); + CHECK_GT(shard_out[op->shard_], 0U); + --me.outstanding_; + --shard_out[op->shard_]; + if (op->req_->Error() != ::eloqstore::KvError::NoError) + { + ++me.read_failed_; + return; + } + me.lat_us_.push_back(Get2NowUs() - op->start_ts_); + }; + + for (auto &op : ops) + { + if (stop.load(std::memory_order_acquire)) + { + break; + } + issue(&op); + } + ReadOperation *done_op = nullptr; + while (!stop.load(std::memory_order_acquire)) + { + if (!me.done_.wait_dequeue_timed(done_op, 10000)) + { + continue; + } + complete(done_op); + if (!stop.load(std::memory_order_acquire)) + { + issue(done_op); + } + } + // The callbacks reference `ops` and `me`; keep both alive + // until every accepted request has completed. + while (me.outstanding_ > 0) + { + me.done_.wait_dequeue(done_op); + complete(done_op); + } + }); + } + + std::this_thread::sleep_for(std::chrono::seconds(total_test_time_sec_)); + stop.store(true, std::memory_order_release); + for (auto &t : thds) + { + t.join(); + } + const double dur_sec = (Get2NowUs() - bench_start) / 1e6; + + std::vector all; + uint64_t read_failures = 0; + uint64_t issue_failures = 0; + for (auto &cl : clients) + { + read_failures += cl.read_failed_; + issue_failures += cl.issue_failed_; + all.insert(all.end(), cl.lat_us_.begin(), cl.lat_us_.end()); + } + const uint64_t successes = all.size(); + std::sort(all.begin(), all.end()); + auto pct = [&](double p) -> uint64_t + { + if (all.empty()) + { + return 0; + } + const size_t idx = + static_cast(p * static_cast(all.size() - 1)); + return all[idx]; + }; + LOG(INFO) << "GET2 finished: clients=" << client_threads + << " inflight=" << inflight << " per_shard_cap=" << per_shard_cap + << " successes=" << successes + << " read_failures=" << read_failures + << " issue_failures=" << issue_failures << " duration=" << dur_sec + << "s QPS:" << std::fixed << std::setprecision(2) + << successes / dur_sec; + LOG(INFO) << "Latency: Min->" << (all.empty() ? 0 : all.front()) + << ", Max->" << (all.empty() ? 0 : all.back()) << ", Mean->" + << (all.empty() ? 0 + : std::accumulate(all.begin(), all.end(), 0ULL) / + all.size()) + << ", p50->" << pct(0.50) << ", p90->" << pct(0.90) << ", p95->" + << pct(0.95) << ", p99->" << pct(0.99) << ", p99.9->" + << pct(0.999) << ", p99.99->" << pct(0.9999); + + // A latency benchmark that reports percentiles over surviving samples + // must not exit success when requests were rejected/failed or nothing + // completed — otherwise a broken run reads as a fast one. main() + // turns this into a nonzero process exit. + if (read_failures != 0 || issue_failures != 0 || successes == 0) + { + failed_ = true; + LOG(ERROR) << "GET2 run FAILED: read_failures=" << read_failures + << " issue_failures=" << issue_failures + << " successes=" << successes; + } +} + void Benchmark::OnRead(::eloqstore::KvRequest *req) { ::eloqstore::ReadRequest *read_req = @@ -611,6 +868,7 @@ void Benchmark::OnRead(::eloqstore::KvRequest *req) // get next key randomly. int8_t iter = obj_iter_type(read_op->bm_->key_pattern_, GET_CMD_IDX); uint64_t key_index = read_obj_gen.get_key_index(iter); + read_op->key_.clear(); read_obj_gen.generate_key(key_index, read_op->key_); @@ -725,6 +983,7 @@ Benchmark::Benchmark(std::string &command, key_pattern_(key_pattern), result_(worker_cnt, this) { + worker_cnt_ = worker_cnt; } bool Benchmark::OpenEloqStore(const eloqstore::KvOptions &kv_options) @@ -848,6 +1107,13 @@ void Benchmark::RunBenchmark() } } } + else if (command_ == "GET2") + { + RunGet2(FLAGS_client_threads, + FLAGS_inflight_per_client, + FLAGS_per_shard_cap); + return; + } else { LOG(ERROR) << "Unsupport command: " << command_; diff --git a/benchmark/eloq_store_bm.h b/benchmark/eloq_store_bm.h index f1fb8ed8..1176482a 100644 --- a/benchmark/eloq_store_bm.h +++ b/benchmark/eloq_store_bm.h @@ -133,17 +133,15 @@ struct ReadOperation explicit ReadOperation(const Benchmark *bm); ReadOperation(const ReadOperation &rhs) = delete; - ReadOperation(ReadOperation &&rhs) - : req_(std::move(rhs.req_)), - key_(std::move(rhs.key_)), - start_ts_(rhs.start_ts_) - { - } + ReadOperation(ReadOperation &&rhs) noexcept = default; req_uptr req_; std::string key_; uint64_t start_ts_{0}; const Benchmark *bm_{nullptr}; + // GET2 mode: owning client and target shard of the in-flight request. + void *client_{nullptr}; + uint32_t shard_{0}; }; class BMResult @@ -220,6 +218,23 @@ class Benchmark void CloseEloqStore(); void RunBenchmark(); + // GET2: dedicated client threads, each keeping `inflight` async reads + // outstanding; optional per-shard outstanding cap bounds the blast + // radius of a stalled shard. + void RunGet2(uint32_t client_threads, + uint32_t inflight, + uint32_t per_shard_cap); + static void OnReadV2(::eloqstore::KvRequest *req); + + /** + * @brief True if the last run recorded request failures or produced no + * successful samples. main() propagates it to a nonzero process exit so + * a broken run cannot masquerade as a fast one. + */ + bool Failed() const + { + return failed_; + } private: static void OnBatchWrite(::eloqstore::KvRequest *req); @@ -239,6 +254,7 @@ class Benchmark std::string command_; size_t total_data_size_{0}; const uint32_t partition_count_{0}; + uint32_t worker_cnt_{0}; // num shard threads (for same-shard mode) uint32_t key_byte_size_{0}; uint32_t value_byte_size_{0}; std::string key_prefix_; @@ -254,6 +270,7 @@ class Benchmark mutable std::vector load_obj_gens_; uint64_t start_ts_{0}; mutable BMResult result_; + bool failed_{false}; // set by RunGet2 on request failure / no samples friend BMResult; friend LoadPartitionsOperation; diff --git a/benchmark/interference_bench.cpp b/benchmark/interference_bench.cpp new file mode 100644 index 00000000..06259d78 --- /dev/null +++ b/benchmark/interference_bench.cpp @@ -0,0 +1,575 @@ +/** + * IO QoS interference benchmark (docs/design/io_qos.md, plan commit 3). + * + * Measures how much a background write/compaction storm degrades foreground + * point-read tail latency, and how the IO QoS knobs (disk_rate_limit_iops, + * rate_bg_ratio, rate_limit_burst_ms, rate_limit_io_unit, max_inflight_io) + * change that. + * + * Phases: + * 1. load — fill P partitions with K keys of ~val_size bytes each. + * val_size defaults to 3000 so one KV fills one 4KB data + * page: key granularity == page granularity, which lets the + * storm control per-file liveness exactly. + * 2. baseline — closed-loop uniform-random point reads at fixed + * concurrency for baseline_secs. No writes. + * 3. mixed — the measured workload: a write-dominated op mix (target + * 90% write key-ops / 10% point reads, --write_read_ratio) + * where completed write batches grant read credits. The + * ratio is a read upper bound: slow readers can make the + * achieved mix more write-heavy, which achieved_write_pct + * reports. Set + * write_read_ratio=0 for the original unthrottled + * reads-vs-storm shape. The writes overwrite a rotating + * strided subset of keys (span of every ratio, default 3 + * of 5, shifted by one each round), keeping every data + * file ~40% live — above file_amplify_factor — so + * compaction continuously relocates live pages through + * 128-page ReadPages bursts. (A full overwrite would leave + * files 100% dead: compaction just drops them and generates + * NO read traffic — see io_qos_impl_plan.md.) + * + * Reports per phase: read QPS and exact p50/p90/p99/p99.9/max latency + * (computed from raw samples, not a sliding window), the storm's write MB/s, + * and per-shard IoQosStats deltas (rate-budget blocks/spend/borrows, io + * window, fdatasync). Greppable one-line summaries are prefixed with + * "RESULT" for sweep scripts. A run exits nonzero if either measured phase + * has errors, missing keys, fewer than --min_read_samples successes, or an + * enabled rate budget records no mixed-phase background spend. + * + * EloqStore options (including the QoS knobs) come from --kvoptions ini, so + * sweeps only vary the ini / flags. See opts_interference.ini. + */ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "coding.h" +#include "eloq_store.h" +#include "utils.h" + +// https://github.com/cameron314/concurrentqueue/issues/280 +#undef BLOCK_SIZE +#include "../external/concurrentqueue/blockingconcurrentqueue.h" + +DEFINE_string(kvoptions, "", "Path to EloqStore options ini"); +DEFINE_uint32(partitions, 4, "number of partitions"); +DEFINE_uint32(keys_per_partition, 20000, "keys per partition"); +DEFINE_uint32(val_size, + 3000, + "value bytes; default ~3000 = one KV per 4KB data page"); +DEFINE_uint32(read_concurrency, 32, "concurrent point reads (closed loop)"); +DEFINE_uint32(baseline_secs, 15, "seconds of read-only baseline phase"); +DEFINE_uint32(storm_secs, 60, "seconds of read + write-storm phase"); +DEFINE_uint32(storm_ratio, 5, "key-stride width of the overwrite pattern"); +DEFINE_uint32(storm_span, 3, "keys overwritten within each stride"); +DEFINE_uint32(storm_batch_keys, 2048, "keys per storm batch-write request"); +DEFINE_uint32(write_read_ratio, + 9, + "completed write key-ops required to grant one mixed-phase " + "read (9 targets at least 90% writes; slow readers can make it " + "more write-heavy). 0 = unthrottled closed-loop reads"); +DEFINE_bool(load, true, "load data first (false reuses an existing store)"); +DEFINE_uint64(min_read_samples, + 1000, + "minimum successful reads required in each measured phase"); + +using namespace std::chrono; + +namespace +{ +constexpr char kTable[] = "ifb"; + +std::atomic g_phase{0}; // 0 load, 1 baseline, 2 mixed, 3 done + +// Read pacing for the mixed phase: every completed storm batch adds its key +// count; issuing one read consumes write_read_ratio credits, allowing at most +// one read per ratio completed writes. Slow readers can make the actual mix +// more write-heavy; achieved_write_pct reports it. +std::atomic g_read_credits{0}; + +bool TryConsumeReadCredits() +{ + const int64_t need = FLAGS_write_read_ratio; + int64_t cur = g_read_credits.load(std::memory_order_relaxed); + while (cur >= need) + { + if (g_read_credits.compare_exchange_weak( + cur, cur - need, std::memory_order_relaxed)) + { + return true; + } + } + return false; +} + +void EncodeKey(char *dst, uint64_t key) +{ + eloqstore::EncodeFixed64(dst, eloqstore::ToBigEndian(key)); +} + +std::string MakeKey(uint64_t key) +{ + std::string s; + s.resize(sizeof(uint64_t)); + EncodeKey(s.data(), key); + return s; +} + +// ---------------------------------------------------------------- readers + +struct Reader +{ + eloqstore::ReadRequest request_; + char key_[sizeof(uint64_t)]; + uint64_t start_us_{0}; + int issue_phase_{0}; +}; + +struct PhaseLatencies +{ + std::vector samples; // microseconds + uint64_t not_found{0}; + uint64_t errors{0}; +}; + +uint64_t Percentile(std::vector &sorted, double p) +{ + if (sorted.empty()) + { + return 0; + } + size_t idx = static_cast(p * (sorted.size() - 1)); + return sorted[idx]; +} + +void ReportPhase(const char *name, PhaseLatencies &lat, double secs) +{ + std::sort(lat.samples.begin(), lat.samples.end()); + const size_t n = lat.samples.size(); + const uint64_t qps = secs > 0 ? static_cast(n / secs) : 0; + const uint64_t p99 = Percentile(lat.samples, 0.99); + LOG(INFO) << "RESULT phase=" << name << " reads=" << n << " qps=" << qps + << " p50=" << Percentile(lat.samples, 0.50) + << " p90=" << Percentile(lat.samples, 0.90) << " p99=" << p99 + << " p999=" << Percentile(lat.samples, 0.999) + << " max=" << (n ? lat.samples.back() : 0) + << " not_found=" << lat.not_found << " errors=" << lat.errors + << " (latency us)"; +} + +bool ValidatePhase(const char *name, const PhaseLatencies &lat) +{ + bool valid = true; + if (lat.samples.size() < FLAGS_min_read_samples) + { + LOG(ERROR) << name << " phase produced only " << lat.samples.size() + << " successful reads; require at least " + << FLAGS_min_read_samples; + valid = false; + } + if (lat.not_found != 0 || lat.errors != 0) + { + LOG(ERROR) << name << " phase had not_found=" << lat.not_found + << " errors=" << lat.errors; + valid = false; + } + return valid; +} + +/** + * Read driver. Baseline phase: closed loop at FLAGS_read_concurrency. + * Mixed phase with write_read_ratio > 0: reads are additionally gated on + * credits produced by completed storm writes, preventing reads from exceeding + * 1 read per ratio writes. Writers are not paced by readers, so slow readers + * can produce a more write-heavy mix. Each completion is recorded into the + * phase the request was ISSUED in (so a request straddling a phase flip does + * not contaminate the other phase). + */ +void ReadLoop(eloqstore::EloqStore *store, + PhaseLatencies *baseline, + PhaseLatencies *mixed) +{ + moodycamel::BlockingConcurrentQueue finished; + std::vector> readers(FLAGS_read_concurrency); + std::vector idle; + idle.reserve(FLAGS_read_concurrency); + for (uint32_t i = 0; i < FLAGS_read_concurrency; i++) + { + readers[i] = std::make_unique(); + idle.push_back(readers[i].get()); + } + + auto callback = [&finished](eloqstore::KvRequest *req) + { + CHECK(finished.enqueue(reinterpret_cast(req->UserData()))) + << "read completion queue allocation failed"; + }; + + std::mt19937_64 rnd(12345); + auto send_req = [&](Reader *reader) + { + const uint64_t key = rnd() % FLAGS_keys_per_partition; + const uint32_t part = rnd() % FLAGS_partitions; + EncodeKey(reader->key_, key); + reader->request_.SetArgs(eloqstore::TableIdent(kTable, part), + std::string_view(reader->key_, sizeof(key))); + reader->issue_phase_ = g_phase.load(std::memory_order_relaxed); + reader->start_us_ = utils::UnixTs(); + CHECK(store->ExecAsyn(&reader->request_, uint64_t(reader), callback)) + << "read issue rejected; fixed-depth result is invalid"; + }; + + size_t inflight = 0; + while (true) + { + Reader *reader; + if (finished.wait_dequeue_timed(reader, milliseconds(1))) + { + const uint64_t lat = + utils::UnixTs() - reader->start_us_; + PhaseLatencies *dst = reader->issue_phase_ == 1 ? baseline + : reader->issue_phase_ == 2 ? mixed + : nullptr; + if (dst != nullptr) + { + if (reader->request_.Error() == eloqstore::KvError::NoError) + { + dst->samples.push_back(lat); + } + else if (reader->request_.Error() == + eloqstore::KvError::NotFound) + { + dst->not_found++; + } + else + { + dst->errors++; + } + } + inflight--; + idle.push_back(reader); + } + + const int phase = g_phase.load(std::memory_order_relaxed); + if (phase >= 3) + { + if (inflight == 0) + { + break; + } + continue; // drain without re-issuing + } + while (!idle.empty()) + { + if (phase == 2 && FLAGS_write_read_ratio > 0 && + !TryConsumeReadCredits()) + { + break; // wait for storm writes to earn more read credits + } + send_req(idle.back()); + idle.pop_back(); + inflight++; + } + } +} + +// ------------------------------------------------------------ write storm + +struct StormWriter +{ + explicit StormWriter(uint32_t part) : part_(part) + { + } + const uint32_t part_; + eloqstore::BatchWriteRequest request_; + uint64_t next_key_{0}; + uint32_t round_{0}; + uint64_t bytes_written_{0}; + uint64_t keys_written_{0}; + uint32_t batch_keys_{0}; // keys in the currently in-flight batch +}; + +/** + * Build the next storm batch for one partition: ascending keys where + * ((key + round) % ratio) < span. Rotating `round` shifts the surviving 40% + * every pass over the keyspace, so files written by earlier rounds are + * partially — never fully — invalidated, keeping compaction's move pipeline + * busy for the whole phase. + */ +void NextStormBatch(StormWriter &w) +{ + std::vector entries; + entries.reserve(FLAGS_storm_batch_keys); + const uint64_t ts = utils::UnixTs(); + const std::string value(FLAGS_val_size, 'S' + (w.round_ & 7)); + while (entries.size() < FLAGS_storm_batch_keys) + { + if (w.next_key_ >= FLAGS_keys_per_partition) + { + w.next_key_ = 0; + w.round_++; + if (!entries.empty()) + { + // Batch keys must stay sorted: never let a batch span the + // keyspace wrap (it would append low keys after high ones). + break; + } + } + const uint64_t key = w.next_key_++; + if ((key + w.round_) % FLAGS_storm_ratio >= FLAGS_storm_span) + { + continue; + } + entries.emplace_back( + MakeKey(key), value, ts, eloqstore::WriteOp::Upsert); + w.bytes_written_ += FLAGS_val_size + sizeof(uint64_t); + } + w.batch_keys_ = static_cast(entries.size()); + w.keys_written_ += w.batch_keys_; + w.request_.SetArgs(eloqstore::TableIdent(kTable, w.part_), + std::move(entries)); +} + +struct StormTotals +{ + uint64_t bytes{0}; + uint64_t keys{0}; +}; + +/** + * One outstanding batch write per partition (matching the engine's + * per-partition write serialization), re-issued on completion until the + * phase ends. Each completed batch grants its key count as read credits + * (see TryConsumeReadCredits). Returns total bytes/keys submitted. + */ +StormTotals StormLoop(eloqstore::EloqStore *store) +{ + moodycamel::BlockingConcurrentQueue finished; + std::vector> writers(FLAGS_partitions); + for (uint32_t i = 0; i < FLAGS_partitions; i++) + { + writers[i] = std::make_unique(i); + } + auto callback = [&finished](eloqstore::KvRequest *req) + { + CHECK( + finished.enqueue(reinterpret_cast(req->UserData()))) + << "storm completion queue allocation failed"; + }; + + for (auto &w : writers) + { + NextStormBatch(*w); + CHECK(store->ExecAsyn(&w->request_, uint64_t(w.get()), callback)) + << "initial storm issue rejected; fixed-depth result is invalid"; + } + size_t inflight = writers.size(); + while (inflight > 0) + { + StormWriter *w; + finished.wait_dequeue(w); + CHECK(w->request_.Error() == eloqstore::KvError::NoError) + << "storm write failed: " + << eloqstore::ErrorString(w->request_.Error()); + g_read_credits.fetch_add(w->batch_keys_, std::memory_order_relaxed); + if (g_phase.load(std::memory_order_relaxed) >= 3) + { + inflight--; + continue; + } + NextStormBatch(*w); + CHECK(store->ExecAsyn(&w->request_, uint64_t(w), callback)) + << "storm reissue rejected; fixed-depth result is invalid"; + } + StormTotals total; + for (auto &w : writers) + { + total.bytes += w->bytes_written_; + total.keys += w->keys_written_; + } + return total; +} + +// ------------------------------------------------------------------- load + +void Load(eloqstore::EloqStore *store) +{ + const std::string value(FLAGS_val_size, 'L'); + const uint64_t ts = utils::UnixTs(); + for (uint32_t part = 0; part < FLAGS_partitions; part++) + { + for (uint64_t base = 0; base < FLAGS_keys_per_partition; + base += FLAGS_storm_batch_keys) + { + const uint64_t end = std::min( + base + FLAGS_storm_batch_keys, FLAGS_keys_per_partition); + std::vector entries; + entries.reserve(end - base); + for (uint64_t key = base; key < end; key++) + { + entries.emplace_back( + MakeKey(key), value, ts, eloqstore::WriteOp::Upsert); + } + eloqstore::BatchWriteRequest req; + req.SetArgs(eloqstore::TableIdent(kTable, part), + std::move(entries)); + store->ExecSync(&req); + CHECK(req.Error() == eloqstore::KvError::NoError) + << "load failed: " << eloqstore::ErrorString(req.Error()); + } + LOG(INFO) << "loaded partition " << part << " (" + << FLAGS_keys_per_partition << " keys)"; + } +} + +// ------------------------------------------------------------------ stats + +void ReportQosDelta(const char *name, + const eloqstore::IoQosStats &begin, + const eloqstore::IoQosStats &end, + size_t shard) +{ + auto d = [](uint64_t b, uint64_t e) { return e - b; }; + LOG(INFO) + << "RESULT qos phase=" << name << " shard=" << shard + << " fdatasync=" << d(begin.fdatasync_count_, end.fdatasync_count_) + << " fdatasync_us=" << d(begin.fdatasync_us_, end.fdatasync_us_) + << " rate_blocked=" + << d(begin.rate_.blocked_count_, end.rate_.blocked_count_) + << " rate_blocked_us=" + << d(begin.rate_.blocked_us_, end.rate_.blocked_us_) + << " rate_ops=" << d(begin.rate_.admitted_ops_, end.rate_.admitted_ops_) + << " rate_mb=" + << (d(begin.rate_.admitted_bytes_, end.rate_.admitted_bytes_) >> 20) + << " rate_borrowed=" + << d(begin.rate_.borrowed_ops_, end.rate_.borrowed_ops_) + << " bg_rate_blocked=" + << d(begin.bg_rate_.blocked_count_, end.bg_rate_.blocked_count_) + << " bg_rate_blocked_us=" + << d(begin.bg_rate_.blocked_us_, end.bg_rate_.blocked_us_) + << " bg_rate_borrowed=" + << d(begin.bg_rate_.borrowed_ops_, end.bg_rate_.borrowed_ops_) + << " io_hwm=" << end.io_window_hwm_ << " io_blocked=" + << d(begin.io_window_blocked_, end.io_window_blocked_); +} + +} // namespace + +int main(int argc, char *argv[]) +{ + google::ParseCommandLineFlags(&argc, &argv, true); + CHECK_GT(FLAGS_partitions, 0u); + CHECK_GT(FLAGS_keys_per_partition, 0u); + CHECK_GT(FLAGS_read_concurrency, 0u); + CHECK_GT(FLAGS_baseline_secs, 0u); + CHECK_GT(FLAGS_storm_secs, 0u); + CHECK_GT(FLAGS_storm_batch_keys, 0u); + CHECK_GT(FLAGS_min_read_samples, 0u); + CHECK_GT(FLAGS_storm_span, 0u); + CHECK_GT(FLAGS_storm_ratio, FLAGS_storm_span) + << "storm must be a PARTIAL overwrite (span < ratio); a full " + "overwrite leaves files 100% dead and compaction generates no " + "read traffic"; + + eloqstore::KvOptions options; + if (int res = options.LoadFromIni(FLAGS_kvoptions.c_str()); res != 0) + { + LOG(FATAL) << "Failed to parse " << FLAGS_kvoptions << " at " << res; + } + LOG(INFO) << "QoS knobs: max_inflight_read=" << options.max_inflight_read + << " bg_read_ratio=" << options.bg_read_ratio + << " max_inflight_write=" << options.max_inflight_write; + + eloqstore::EloqStore store(options); + if (auto err = store.Start("main", 0); err != eloqstore::KvError::NoError) + { + LOG(FATAL) << "Failed to start store: " << eloqstore::ErrorString(err); + } + + if (FLAGS_load) + { + Load(&store); + } + + const size_t num_shards = store.Options().num_threads; + std::vector qos_start(num_shards); + std::vector qos_mid(num_shards); + std::vector qos_end(num_shards); + + PhaseLatencies baseline, storm_lat; + + // Baseline phase: reads only. + uint64_t mixed_bg_read_pages = 0; + for (size_t s = 0; s < num_shards; s++) + { + qos_start[s] = store.GetIoQosStats(s); + } + g_phase.store(1, std::memory_order_relaxed); + std::thread read_thd(ReadLoop, &store, &baseline, &storm_lat); + std::this_thread::sleep_for(seconds(FLAGS_baseline_secs)); + + // Storm phase: reads + rotating partial-overwrite write storm. + for (size_t s = 0; s < num_shards; s++) + { + qos_mid[s] = store.GetIoQosStats(s); + } + g_phase.store(2, std::memory_order_relaxed); + StormTotals storm_totals; + std::thread storm_thd([&] { storm_totals = StormLoop(&store); }); + std::this_thread::sleep_for(seconds(FLAGS_storm_secs)); + + g_phase.store(3, std::memory_order_relaxed); + storm_thd.join(); + read_thd.join(); + for (size_t s = 0; s < num_shards; s++) + { + qos_end[s] = store.GetIoQosStats(s); + } + + ReportPhase("baseline", baseline, FLAGS_baseline_secs); + ReportPhase("mixed", storm_lat, FLAGS_storm_secs); + const uint64_t read_ops = storm_lat.samples.size(); + const double achieved_write_pct = + storm_totals.keys + read_ops > 0 + ? 100.0 * storm_totals.keys / (storm_totals.keys + read_ops) + : 0.0; + LOG(INFO) << "RESULT mixed write_mb_per_sec=" + << (storm_totals.bytes >> 20) / + std::max(1, FLAGS_storm_secs) + << " write_mb=" << (storm_totals.bytes >> 20) + << " write_key_ops=" << storm_totals.keys + << " read_ops=" << read_ops << " achieved_write_pct=" + << static_cast(achieved_write_pct + 0.5); + for (size_t s = 0; s < num_shards; s++) + { + ReportQosDelta("baseline", qos_start[s], qos_mid[s], s); + ReportQosDelta("mixed", qos_mid[s], qos_end[s], s); + mixed_bg_read_pages += qos_end[s].bg_rate_.admitted_ops_ - + qos_mid[s].bg_rate_.admitted_ops_; + } + + const bool baseline_valid = ValidatePhase("baseline", baseline); + const bool mixed_valid = ValidatePhase("mixed", storm_lat); + bool interference_valid = true; + if (options.disk_rate_limit_iops != 0 && mixed_bg_read_pages == 0) + { + LOG(ERROR) << "mixed phase recorded no background rate-budget " + "spend; the intended write/compaction interference " + "was not exercised"; + interference_valid = false; + } + const bool valid = baseline_valid && mixed_valid && interference_valid; + LOG(INFO) << "RESULT validation=" << (valid ? "pass" : "fail"); + + store.Stop(); + return valid ? 0 : 2; +} diff --git a/benchmark/main.cpp b/benchmark/main.cpp index a75ed366..46545e2b 100644 --- a/benchmark/main.cpp +++ b/benchmark/main.cpp @@ -36,6 +36,14 @@ DEFINE_uint32(batch_size, 64, "The batch size of one write request (MB) (default: 64)."); DEFINE_string(storage, "eloqstore", "The storage used to store the data."); +DEFINE_uint32(client_threads, 4, "GET2: number of client threads."); +DEFINE_uint32(inflight_per_client, + 125, + "GET2: async requests each client keeps in flight."); +DEFINE_uint32(per_shard_cap, + 0, + "GET2: max outstanding per shard per client (0 = unlimited); " + "bounds the blast radius of a stalled shard."); DEFINE_uint32(request_cnt, 32, "The number of concurrent read requests (default: 32)"); @@ -78,6 +86,7 @@ DEFINE_uint64(subcompactions, int main(int argc, char *argv[]) { + int exit_code = 0; FLAGS_logtostderr = true; google::InitGoogleLogging("EloqStore_benchmark"); @@ -133,6 +142,10 @@ int main(int argc, char *argv[]) // Run the test bench_mark.RunBenchmark(); + if (bench_mark.Failed()) + { + exit_code = 1; + } // Shutdown the eloq store bench_mark.CloseEloqStore(); @@ -188,5 +201,5 @@ int main(int argc, char *argv[]) } google::ShutdownGoogleLogging(); - return 0; + return exit_code; } \ No newline at end of file diff --git a/benchmark/opts_interference.ini b/benchmark/opts_interference.ini new file mode 100644 index 00000000..43d6b28c --- /dev/null +++ b/benchmark/opts_interference.ini @@ -0,0 +1,38 @@ +# EloqStore options for interference_bench (docs/design/io_qos.md M4). +# +# Append mode with small (8MB) data files so the rotating partial-overwrite +# storm pushes per-file space amplification past file_amplify_factor quickly, +# keeping compaction's move pipeline (background ReadPages bursts) busy. +# +# Sweep the M4 device rate-limit knobs below (see docs/design/io_qos.md +# "Sizing contract" and "Validation plan"): +# disk_rate_limit_iops in {200000, 260000, 275000, 290000} (~95% of the +# fio-measured device ceiling is the recommended setting) +# rate_bg_ratio in {10, 25, 50} (background share of the budget) +# rate_limit_burst_ms in {1, 2, 4} +# A/B the limiter by setting disk_rate_limit_iops = 0 (off) versus a +# calibrated value; the deprecated count knobs (max_inflight_read, +# bg_read_ratio, max_inflight_write) no longer affect device admission. + +# Note: run interference_bench with --partitions >= num_threads, or some +# shards sit idle and their RESULT qos lines read zero. + +[run] +num_threads = 4 +buffer_pool_size = 400MB +fd_limit = 5000 +num_retained_archives = 0 +skip_verify_checksum = true + +# --- M4 device rate limit under test --- +disk_rate_limit_iops = 275000 +rate_bg_ratio = 25 +rate_limit_burst_ms = 2 +rate_limit_io_unit = 4KB +# max_inflight_io = 0 # optional class-blind in-flight window + +[permanent] +store_path = /tmp/eloqstore_interference +data_page_size = 4KB +data_file_size = 8MB +data_append_mode = true diff --git a/db_stress/README.md b/db_stress/README.md index e1783231..7bcc0a01 100644 --- a/db_stress/README.md +++ b/db_stress/README.md @@ -66,8 +66,8 @@ There are two groups of knobs: - `--data_append_mode`, `--pages_per_file_shift`, `--data_page_size` - `--buffer_pool_size`, `--io_queue_size`, `--fd_limit` - `--manifest_limit`, `--init_page_count` - - `--max_inflight_write`, `--max_write_batch_pages` + - `--max_inflight_write` (`--max_write_batch_pages` is deprecated and + ignored) - `--file_amplify_factor`, `--local_space_limit`, `--reserve_space_ratio` - `--overflow_pointers`, `--data_page_restart_interval`, `--index_page_restart_interval` - diff --git a/db_stress/crash_test.py b/db_stress/crash_test.py index ec317cbd..cb4a04e6 100755 --- a/db_stress/crash_test.py +++ b/db_stress/crash_test.py @@ -144,7 +144,8 @@ def randomize_dynamic_params(): "fd_limit": random.choice([5000, 10000, 15000]), "io_queue_size": random.choice([2048, 4096, 8192]), "max_inflight_write": random.choice([2048, 4096, 8192]), - "max_write_batch_pages": random.choice([32, 64, 128]), + # Deprecated and ignored; keep a fixed value for CLI compatibility. + "max_write_batch_pages": 64, # "coroutine_stack_size": lambda: random.choice([1<<13, 1<<14, 1<<15]), "file_amplify_factor": random.choice([2]), "reserve_space_ratio": random.choice([50, 100, 150, 200]), diff --git a/db_stress/db_stress_gflags.cpp b/db_stress/db_stress_gflags.cpp index 3c25d5bc..d9858cef 100644 --- a/db_stress/db_stress_gflags.cpp +++ b/db_stress/db_stress_gflags.cpp @@ -92,7 +92,9 @@ DEFINE_uint32(pages_per_file_shift, 11, "nums of filepage shift"); DEFINE_uint32(max_inflight_write, 4096, "Max amount of inflight write IO per thread"); -DEFINE_uint32(max_write_batch_pages, 64, "max pages per write batch"); +DEFINE_uint32(max_write_batch_pages, + 64, + "Deprecated compatibility option; ignored"); DEFINE_uint32(num_retained_archives, 0, "limit number of retained archives"); DEFINE_uint32(archive_interval_secs, 86400, "archive time interval in secs"); DEFINE_uint32(max_archive_tasks, 256, "max running archive tasks"); diff --git a/docs/architecture/02-runtime-and-lifecycle.md b/docs/architecture/02-runtime-and-lifecycle.md index 171dd5c7..0a1dd35c 100644 --- a/docs/architecture/02-runtime-and-lifecycle.md +++ b/docs/architecture/02-runtime-and-lifecycle.md @@ -105,9 +105,18 @@ state: - **Topology**: `num_threads` (shards), `fd_limit`, `io_queue_size`, `buffer_pool_size` (index-page cache per shard; data pages share it only if `enable_data_page_cache`), `root_meta_cache_size` (global RootMeta LRU). -- **Write shaping**: `max_write_batch_pages`, `max_write_concurrency`, +- **Write shaping**: `max_write_concurrency`, `write_buffer_size`/`write_buffer_ratio` (append-mode aggregation), - `manifest_limit` (snapshot-rotation threshold). + `manifest_limit` (snapshot-rotation threshold), `max_inflight_write` + (write request-pool sizing). + (`max_write_batch_pages` is deprecated and ignored.) +- **IO QoS** (doc 07, `docs/design/io_qos.md` M4): `disk_rate_limit_iops` + /`disk_rate_limit_mbps` (per-disk device rate limit, on by default, + divided across shards), `rate_bg_ratio` (background share of the rate — + the tail-predictability policy knob), `rate_limit_burst_ms`, + `rate_limit_io_unit` (write ops quantum, minimum 4KB), and `max_inflight_io` (optional + class-blind in-flight command window). The former count budgets + `max_inflight_read`/`bg_read_ratio` are deprecated no-ops. - **Cloud**: `cloud_provider/endpoint/region/keys`, `local_space_limit` + `reserve_space_ratio` (cache budget), `max_cloud_concurrency`, `cloud_request_threads`, `allow_reuse_local_caches`, `prewarm_cloud_cache`. diff --git a/docs/architecture/04-execution-model.md b/docs/architecture/04-execution-model.md index f3542118..589920a7 100644 --- a/docs/architecture/04-execution-model.md +++ b/docs/architecture/04-execution-model.md @@ -23,24 +23,46 @@ shard context without parameter plumbing. `Shard::WorkLoop()` (normal build) repeats: -1. `io_mgr_->Submit()` — flush prepared io_uring SQEs. -2. `io_mgr_->PollComplete()` — reap CQEs; each completion either finishes a - blocked task's I/O (`FinishIo`) or processes a background write request. - Cloud/standby ready-queues are drained here too. -3. `ExecuteReadyTasks()` — resume coroutines from `ready_tasks_`, then (when - the normal queue is empty) `low_priority_ready_tasks_` (background - compaction/GC yield here to protect foreground latency). +1. `io_mgr_->Submit()` — refill the device rate budget (`RefillAndWake`, its + only wake source) and enter the kernel. The ring uses + `IORING_SETUP_DEFER_TASKRUN`, so this entry is also what delivers CQEs. +2. `io_mgr_->PollComplete()` — reap CQEs (pure user space: `peek_cqe` + + `for_each_cqe`); each completion either finishes a blocked task's I/O + (`FinishIo`) or processes a background write request. Cloud/standby + ready-queues are drained here too. +3. `PromoteReadyDelayedReopenRequests()`. 4. Dequeue up to 128 new `KvRequest`s from the MPSC `requests_` queue (blocking with 100 ms timeout only when fully idle) and feed each to `OnReceivedReq`. +5. `ExecuteReadyTasks()` — resume coroutines from `ready_tasks_`, then (when + the normal queue is empty) `low_priority_ready_tasks_` (background + compaction/GC yield here to protect foreground latency). +6. `io_mgr_->FlushSubmit()` — issue the SQEs this round prepared. + +**The order is load-bearing.** Four producers feed `ready_tasks_` — rate-budget +grants (step 1), I/O completions (step 2), delayed reopens (step 3), and new +requests (step 4) — and all of them are placed before the single +`ExecuteReadyTasks`, so work admitted in a round runs in that same round. +`FlushSubmit` then closes the loop on the output side: without it, SQEs +prepared in step 5 would wait for the *next* round's `Submit`, which in module +mode is a full external scheduling quantum of dead device time on every I/O +hop. `FlushSubmit` deliberately does not touch `consecutive_skipped_submits_` +(the DEFER_TASKRUN forced-enter safety net stays owned by `Submit`) and is a +no-op when the round prepared nothing. Exit: when the store is stopping and the shard is idle. Teardown runs on the shard thread: `TaskManager::Shutdown()` → `PageManager::Shutdown()` → `io_mgr_->Stop()` (tasks may hold page pins, so task state dies first). In the module build (`ELOQ_MODULE_ENABLED`, doc 10) the same logic is exposed -as `WorkOneRound()` and an external runtime drives it; `IsIdle()` tells the -runtime whether the shard needs another round. +as `WorkOneRound()` (driven by `EloqStoreModule::Process`) and an external +runtime drives it; `IsIdle()` tells the runtime whether the shard needs +another round. It runs the **same step order**, which matters most there: the +gap between rounds is an external scheduling decision rather than a few +microseconds, so both a delayed flush and mis-ordered admission cost a full +quantum. The only structural difference is that the request *dequeue* happens +first because the idle-round test depends on its count; admission +(`OnReceivedReq`) still runs at step 4, after `PollComplete`. ## Tasks are pooled coroutines @@ -59,6 +81,21 @@ Scheduling primitives: - `WaitingZone` / `WaitingSeat` / `Mutex` — intra-shard wait lists (no real locks; they park/wake coroutines). Used for FD open/close exclusion, pool exhaustion waits, upload completion, etc. +- Rate-budget waits (`RateBudget::Acquire`, `async_io_manager.h`) — tasks park + on a per-class `WaitingZone` when admitting their page IO would drive the + shard's device rate budget (`disk_rate_limit_iops`/`disk_rate_limit_mbps`, + M4) non-positive. Wakes are **refill-driven**, not completion-driven: the + once-per-loop `RefillAndWake` (peek-and-grant — it charges the FIFO head's + recorded cost before waking it) is the only wake source, so waiter progress + depends only on the shard loop running, never on another task completing. + Background tasks (`KvTask::IsBackground()`: BatchWrite, BackgroundWrite, + EvictFile, Prewarm) draw from the background class share (`rate_bg_ratio`) + on a separate FIFO zone; foreground may borrow background's idle surplus but + background never borrows foreground's, so foreground's share is a hard + guarantee. The optional `max_inflight_io` window is the only occupancy cap + that releases per CQE. See `docs/design/io_qos.md` (M4); the acquire order is + FD/mutex → pools/buffers → rate budget → window → SQE, with no voluntary + yield after admission. (The former `IoBudget` count budgets are retired.) `TaskManager` keeps one free-list pool per task type (`BatchWriteTask`, `BackgroundWrite`, `ReadTask`, `ScanTask`, `ListObjectTask`, diff --git a/docs/architecture/07-io-stack.md b/docs/architecture/07-io-stack.md index 89bfb23a..543e3959 100644 --- a/docs/architecture/07-io-stack.md +++ b/docs/architecture/07-io-stack.md @@ -12,7 +12,7 @@ Source: `include/async_io_manager.h`, `src/async_io_manager.cpp`, `AsyncIoManager` is the shard-facing storage interface. One instance per shard, chosen by `AsyncIoManager::Instance` from the store mode: -``` +```text AsyncIoManager (abstract: ReadPage/WritePage/ReadSegments/Manifest ops/...) ├── MemStoreMgr store_path empty — pages and manifests in RAM (tests) └── IouringMgr local filesystem over io_uring — the substrate @@ -37,6 +37,35 @@ Responsibilities: - **Page I/O** — `ReadPage`/`ReadPages` (batched, into pool buffers, fixed reads when the buffer is registered), `WritePage`. `ConvFilePageId` splits a `FilePageId` into `(file_id, offset)` by `pages_per_file_shift`. +- **Device rate budget** (`RateBudget`, see `docs/design/io_qos.md` M4) — + per-shard token buckets metering device **ops/sec** and **bytes/sec**, + refilled lazily from the shard TSC clock once per event-loop iteration + (`RefillAndWake` at the top of `Submit()`) and spent at the page-IO acquire + sites (`ReadPage`/`ReadPages` cost 1 op + one page of bytes; `WritePage` and + `SubmitMergedWrite` cost `WriteRateOps(bytes)` = `ceil(bytes / + rate_limit_io_unit)` ops + `bytes`; `fdatasync` 1 op). Admission is **debt**: + a task waits until the balance is positive, then subtracts its full cost + (may go negative), so the long-run rate is exact and an IO larger than the + bucket never deadlocks. There is **no completion-time release** — a rate + budget meters issuance, so tokens are spent by sending and only time (refill) + restores them. The budget is **partitioned by class**: foreground buckets + refill at `(100 − rate_bg_ratio)%`, background (background-task reads and all + write-path IO) at `rate_bg_ratio%`. Foreground may borrow background's idle + surplus (background never borrows); admission is peek-and-grant — the refill + charges the FIFO head's recorded cost before waking it. Per-disk limits + (`disk_rate_limit_iops`/`disk_rate_limit_mbps`, on by default) are divided + across shards (`× store paths / num_threads`). A separate optional + class-blind in-flight command window (`max_inflight_io`, off by default) is + the only occupancy-style cap and *does* release per CQE in `PollComplete`; + it charges 1 per page IO, `ceil(len / 256KB)` per merged write, and 1 per + batch fsync (`FdatasyncFiles`, tagged `BaseReqFsync` so the release can + tell them from exempt metadata ops). Other metadata, manifest, bulk + file/snapshot paths, and segment IO are exempt from both mechanisms. + `GetIoQosStats()` (also `EloqStore::GetIoQosStats(shard_id)`) exposes the + per-class rate counters (blocked count/us, admitted ops/bytes, borrowed ops) + and the window's in-flight/high-water/blocked gauges. + The M1/M2 count budgets (`IoBudget`, `max_inflight_read`/`bg_read_ratio`) + are **retired**; `max_inflight_write` now only sizes the write request pools. - **FD cache** — `LruFD` per (partition, `TypedFileId`), doubly-linked LRU bounded by the shard's fd budget; `EvictFD` closes idle descriptors. Open/close exclusion per FD via a coroutine `Mutex`. Data files open with diff --git a/docs/architecture/08-data-lifecycle.md b/docs/architecture/08-data-lifecycle.md index 7b4606ec..bf6a5baa 100644 --- a/docs/architecture/08-data-lifecycle.md +++ b/docs/architecture/08-data-lifecycle.md @@ -67,6 +67,14 @@ user writes. Scheduled via shard pending-sets (`AddPendingCompact/TTL/FileGc/LocalGc`) which enqueue the embedded singleton requests in each `PendingWriteQueue`. +All of these run as background tasks (`KvTask::IsBackground()`), so their +data-page reads — compaction move batches in particular — and all their page +writes are charged against the **background share** of the device rate budget +(`rate_bg_ratio`, doc 07 / `docs/design/io_qos.md` M4) and cannot crowd out +foreground reads at the device. A pure-write/compaction phase with no +concurrent foreground reads therefore runs at `rate_bg_ratio` of the device +rate by design (foreground does not lend to background). + - **Compaction** (`Compact()`, append mode only): one `MakeCowRoot`; rewrite pass over under-utilized data files (live pages re-written to the tail, per-file utilization re-checked against `file_amplify_factor`), then over diff --git a/docs/design/io_qos.md b/docs/design/io_qos.md new file mode 100644 index 00000000..59847a9c --- /dev/null +++ b/docs/design/io_qos.md @@ -0,0 +1,658 @@ +# EloqStore IO QoS: Foreground/Background IO Isolation + +## Summary + +EloqStore runs foreground tasks (ReadTask, ScanTask) and background tasks +(BatchWriteTask, BackgroundWrite/compaction, file GC) as coroutines on a +per-shard thread. CPU-side prioritization works: background tasks yield to a +low-priority ready queue and the scheduler bounds low-priority slices to +`max_processing_time_microseconds`. However, experiments show background tasks +still significantly degrade foreground read latency. + +PR #455 (`perf(compaction/gc)`) addressed the CPU half of this problem: the +time-budgeted cooperative yield (`MaybeYield()` / +`eloqstore_yield_budget_us`, default 20µs) bounds how long any single +compaction/GC coroutine segment holds the worker thread, and GC unlink/close +SQE preparation is chunked into 128-op batches. What remains — and what this +document addresses — was that **CPU priority was lost at the IO boundary**. +Before M1/M2, one 20µs slice could still burst up to 128 page reads (compaction +move batches, `DoCompactDataFile`) plus writes into the io_uring ring, and +nothing distinguished those requests from foreground reads at the ring or +device level. Reads were not budgeted at all; writes were budgeted only +per-task. Yielding controlled when background IO was *issued*, not how much of +it queued at the device once issued. + +**Current state (2026):** the shipped mechanism is **M4**, per-shard device +rate limiting (`RateBudget`; see the M4 section below). M1/M2 (in-flight +count budgets) and M3 (a separate background write bytes/sec limiter) were +the original design and are documented below as design record, but M1/M2 are +**retired** in the code and M3 is **subsumed** by M4's byte bucket. The +historical mechanism sketch: + +- **M1** *(retired)*: per-shard caps on in-flight page IO, separate for reads + and writes. +- **M2** *(retired; the FG/BG class model survives in M4's `rate_bg_ratio`)*: + foreground/background classification with a background sub-budget on the + read cap. +- **M3** *(subsumed by M4)*: a bytes/sec rate limiter on background writes. +- **M4** *(current)*: per-shard token buckets metering device ops and bytes + per second, partitioned foreground/background, replacing M1/M2 — the tail + on rate-metered cloud disks is set by the hypervisor's rate limiter, which + a concurrency cap cannot address. + +All caps and limits in this document are **per shard**, consistent with the +rest of EloqStore. + +Related maintained documentation: `docs/architecture/04-execution-model.md` +(scheduler, ready queues, yield discipline), `07-io-stack.md` (IouringMgr, +page/segment IO, pools), `08-data-lifecycle.md` (compaction/GC cadence). +Implementing this design requires updating those docs in the same change +(see CLAUDE.md). + +## Baseline and Current Mechanisms + +### Pre-QoS baseline + +The table and gaps below describe the baseline before M1/M2, not the current +implementation. + +| Baseline mechanism | Location | Scope | +|---|---|---| +| Two ready queues (high / low priority), 400µs low-priority slice | `Shard::ExecuteReadyTasks`, `KvTask::YieldToLowPQ` | CPU only | +| Time-budgeted cooperative yield: `MaybeYield()` once a coroutine segment exceeds `eloqstore_yield_budget_us` (20µs) — added by #455 | `MaybeYield`, `Shard::CurResumeElapsedUs` | CPU only | +| GC unlink/close SQE prep chunked into 128-op batches, `WaitIo` per chunk — added by #455 | `IouringMgr::CloseFiles` / `DeleteFiles` | Metadata IO burst size | +| Per-write-task in-flight cap: `WaitWrite()` when `inflight_io_ >= max_write_batch_pages` (32) | `WriteTask::WritePage` | Writes only, per task | +| `WriteReqPool` sized by `max_inflight_write` (default 32768) | `IouringMgr` | Writes, effectively unbounded | +| Segment compaction: ≤ `max_segments_batch` (8) in-flight 256KB segments per batch, buffers shared with foreground via `GlobalRegisteredMemory`; yields every `segment_compact_yield_every` segments | `BackgroundWrite::DoCompactSegmentFile` | Segment IO, per task | +| SQ ring capacity `io_queue_size` (4096); `GetSQE` blocks on full ring | `IouringMgr::GetSQE`, `waiting_sqe_` | Backpressure, not QoS | +| `max_write_concurrency` bounds concurrent write tasks (incl. compaction: `GetBackgroundWrite` returns null at the limit) | `TaskManager` | Default unlimited in local mode | +| Cloud slots (`max_cloud_concurrency`), cloud buffer pool | `CloudStoreMgr` | Cloud HTTP, not disk | + +### Baseline gaps addressed by M1/M2 + +1. **Reads had no budget anywhere.** Page compaction + (`DoCompactDataFile`) issues bursts of up to `max_read_pages_batch` (128) + page reads per move batch through the same `ReadPages` path as foreground + reads. +2. **There was no shard-global in-flight IO counter.** Only per-task + `inflight_io_` and `prepared_sqe_` existed. The effective global bounds + (4096 SQEs, 32K + in-flight writes) are far beyond the queue depth at which NVMe read + latency degrades. +3. **There was no FG/BG tagging of IO.** SQEs were indistinguishable once + submitted; `sqe->ioprio` was never set. A background task scheduled for one 20µs + slice can leave 128+ requests queued at the device ahead of every + subsequent foreground read — the yield discipline of #455 bounds CPU + occupation per slice, not the IO submitted within it. + +The current M1/M2 implementation uses a per-shard default of 64 configured +data pages for reads and a 25% background read slice. The write budget is +available but retains its effectively-unbounded 32768-page default; 512 remains +an explicit opt-in value. The old `max_write_batch_pages` throttle has no +effect; M3 is still deferred. + +## Device-Level Rationale + +EloqStore's pattern in append mode is **sequential writes vs. random reads**, +which makes the interference mechanism clean: + +- Reads and writes are asymmetric on NVMe. A 4KB write is acknowledged from + the controller buffer (~10–20µs); a 4KB read must touch NAND (~60–100µs). + The deferred cost of writes is NAND program time (~1–3ms per TLC program) + and die occupancy. +- **Die collisions dominate.** A random read landing on a die that is + mid-program waits for it (unless the drive supports program-suspend). To + first order, if background write throughput is fraction ρ of the device's + sustained write bandwidth, then P(read collides) ≈ ρ, and collided reads + pay ~0.5–1.5ms. Median read latency stays flat; tail latency degrades + roughly linearly with write **bytes/sec** — not with in-flight write count. +- Sequential appends keep drive-internal write amplification near 1 + (whole-file GC invalidates large contiguous regions), so there is no + hidden GC multiplier — provided deleted file space is actually TRIMmed. + +Consequences for the design: + +- An in-flight cap (M1/M2) bounds burst queueing at the ring/device, but the + control that maps to the physics of sustained interference is a **write + bytes/sec budget** (M3). +- **Pace, don't burst**: the same average MB/s issued as evenly spaced 1MB + merged writes produces better read tails than periodic multi-MB bursts. + Keep the 1MB merge unit; throttle frequency, not size. + +## Design + +### M1: Per-shard in-flight page-IO caps, reads and writes separate + +> **Superseded (2026-07-22).** The count budgets shipped, were validated, +> and were then retired in favor of M4: on rate-metered cloud disks the +> tail is set by the hypervisor's rate limiter, which a concurrency cap +> cannot address (measured: properly-sized count caps recovered a fraction +> of the rate budget's result), and the write cap could not bind below one +> merged write buffer. `max_inflight_read` / `bg_read_ratio` are deprecated +> no-ops; `max_inflight_write` reverted to write request-pool sizing; +> instantaneous depth bounding, if wanted, is the single class-blind +> `max_inflight_io` window. The sections below are kept as design record. + +Two per-shard counters of in-flight **page** IO (in configured +`data_page_size` units), with independent caps, enforced at the page-IO entry +points of `IouringMgr`. +Reads and writes are deliberately **not** mixed in one budget: they are +different device resources (reads need deep queue depth for IOPS; writes +saturate bandwidth at shallow depth and their interference tracks sustained +bytes/sec), so a shared cap would couple two knobs that must be tuned +separately. + +- Read budget (`inflight_read_pages_` ≤ `max_inflight_read`): + `ReadPage` / `ReadPages`, cost 1 per page. +- Write budget (`inflight_write_pages_` ≤ `max_inflight_write`, option + redefined — see Interaction with Existing Knobs): + `WritePage`, cost 1; `SubmitMergedWrite`, cost = bytes rounded up to + `data_page_size` (a 1MB merged write counts as 256 pages at the default 4KB + page size), so the cap means the same thing in append and non-append mode. +Segment IO (`ReadSegments` / `WriteSegments`, zero-copy large values) is +**out of scope** — see Non-Goals. + +Metadata operations (open, statx, rename, unlink, mkdir), manifest IO, and bulk +file/snapshot paths (`ReadFile`, `ReadFilePrefix`, `WriteSnapshot`) are exempt; +metadata burst size is already bounded by the 128-op chunking from #455. +`fdatasync` is not counted initially but is instrumented (see Evaluation). + +Enforcement follows the existing idiom, per class (read or write). The +in-flight counter holds only previously admitted, not-yet-completed IO; each +request is counted exactly once, from admission to completion. For a new +request of size `cost` (page-units, not yet counted): + +```text +acquire(class, cost): // at the page-IO entry point + while inflight[class] != 0 and inflight[class] + cost > cap[class]: + wait on waiting_zone[class] (FIFO) + inflight[class] += cost // admitted; prep SQE(s) and submit + +release(class, cost): // in PollComplete, on completion + inflight[class] -= cost + wake waiters of that class +``` + +Blocking-on-acquire is preferred over yield-and-retry: it is cheaper and +guarantees FIFO fairness. + +Rules: + +- Acquire the budget immediately before SQE preparation, with **no yield + points between acquire and submit**. +- Release happens in `PollComplete`, which runs regardless of task + scheduling, so budget release never depends on the blocked task running — + no deadlock. Paths that hold budget and then block on another pool (e.g. + write buffer pool) must acquire in a consistent order; audit during + implementation. + +### M2: Foreground/background classification and BG sub-budget + +> **Superseded (2026-07-22).** The FG/BG classification survives — it is +> the class model of the M4 rate budget (`rate_bg_ratio`) — but the +> count-domain sub-budget described here is retired with M1. + +Every page IO is issued from a `KvTask` (`ThdTask()`), so classification is a +task-type predicate. Add `KvTask::IsBackground()`: + +- Background: `BatchWrite`, `BackgroundWrite`, `EvictFile`, `Prewarm`. +- Foreground: `Read`, `Scan`, `ListObject`, `ListStandbyPartition`, `Reopen`. + +Do **not** reuse `ReadOnly()` — `EvictFile` and `Prewarm` are read-only but +background. They do not currently issue budgeted page reads: local GC uses +`ReadFile`, while prewarm/download uses whole-file bulk IO; both remain exempt. + +The FG/BG split only applies to the **read** budget. All page writes are +issued by write tasks (`BatchWrite`, `BackgroundWrite`), which are background +by the classification above — so the write cap of M1 *is* the background +write budget, and needs no further split. (If a genuinely foreground write +class ever appears, split the write budget then.) + +Read budget structure: + +- Foreground reads: `inflight_read_pages_ ≤ max_inflight_read`. +- Background page reads (compaction move batches and batch-write tree-traversal + reads): additionally + `bg_inflight_read_pages_ ≤ bg_read_limit` (a fraction of + `max_inflight_read`, e.g. 25%). + +Background never exceeds its sub-budget. Foreground can consume the entire +read budget **while background has no pending demand**; once a background +acquisition enters the wait path, its unused entitlement +(`bg_read_limit − bg_inflight`) stays reserved through admission, including +the wake-to-admit gap — new foreground admissions leave it alone. Without the +reservation, sustained foreground saturation would starve background +forever (every freed unit would be re-acquired by a foreground waiter and +compaction could never bootstrap its share), letting space amplification +grow unboundedly. With it, the ratio is genuinely maintained under +contention: background ramps to its slice, foreground keeps the rest, and +foreground reclaims the whole budget the moment background demand drains. + +Two read `WaitingZone`s (one per class, FIFO within class). On release, +background waiters are woken first while the sub-budget has room (those +units are reserved for them anyway), and foreground is **always** woken as +well — not merely handed leftover credits. The leftover-only scheme shipped +first and starved the foreground class under write storms (found 2026-07-11 +on Azure NVMe): background forms a saturated treadmill whose queue never +empties, so every release re-donates its credit to background; once any +foreground read queued (admission is FIFO-behind-waiters) and the last +foreground in-flight's credit landed in a background dip, the foreground +class had **no remaining wake source** until background's queue drained — +observed as multi-second foreground gate stalls (p999 120–312 ms, gate +waits up to 3.7 s). Over-waking is safe by construction: woken tasks +re-check the admission condition and re-wait. + +The ratio starts **static**. An adaptive policy (shrink `bg_read_limit` +toward a floor when foreground waiters exist, grow toward the full budget +when the shard is idle) is a possible follow-up once counters justify it. + +### M3: Background write rate limiter (follow-up) + +A per-shard token bucket in bytes, refilled at `bg_write_rate_limit` bytes/sec: + +- `WritePage` / `SubmitMergedWrite` from background tasks debit the bucket + and block on a `WaitingZone` when empty. +- Bucket capacity is small (a few refill intervals worth) so a full bucket + cannot discharge as a burst; this paces merged writes evenly. +- 0 = disabled (default). Enable after the calibration sweep (below) shows + the tail-latency knee for the target device. + +M3 is deferred until experiments confirm that in-flight capping alone (M1+M2) +is insufficient — which the die-collision model predicts for sustained +compaction, since interference tracks bytes/sec rather than queue depth. + +**Status 2026-07-21: subsumed by M4 below.** The Azure campaign confirmed the +prediction — interference tracks rate, not queue depth — and additionally +showed that on cloud disks the dominant tail source is the *hypervisor's own +rate limiter*, which M4 addresses for all IO classes, not only background +writes. + +### M4: Per-shard device rate limiting (ops/sec + bytes/sec) + +**Motivation (measured 2026-07-21, Azure L-series local NVMe).** Cloud disks +are provisioned rate limits, not devices: `/dev/nvme1n1` enforced ~275K IOPS, +holding the overflow fraction of IOs for a quantized ~3 ms (bimodal latency: +90 µs or ~3 ms, nothing between). A read-only workload driving the disk at +its ceiling showed p99.9 = 3.3 ms with all engine stages clean; deep-queue +fio reproduced the plateau (6.7 ms) at the same 275K ceiling, and sub-ceiling +fio showed the true device tail is ~200 µs. AWS documents the same +enforcement for Nitro instance-store NVMe ("performance exceeded" counters); +GCP Local SSD limits scale with instance shape. Conclusion: **the engine must +own the queue** — admit IO below the provisioned ceiling so waiting happens +in user space (FIFO, foreground-prioritized, observable) instead of in the +hypervisor's limiter (random ~ms quantized holds, no priority). + +Count caps (M1) cannot express this robustly: the count that keeps the rate +at the ceiling is `rate × latency`, and the sweet spot is narrow — on the +test disk, 24 in-flight store-wide eliminated throttle holds (p99.9 +3,252 → 419 µs, −8% throughput) while 32 in-flight retained nearly the full +throttle tail and 16 forfeited 40% of throughput. A rate budget hits the +target directly and stays correct as latency shifts with the workload mix. + +**Budget derivation — simple division.** The deployment knows the per-disk +provisioned limits, the number of disks (`store_path` count), and the number +of shards (`num_threads`). Each shard's rate budget is: + +``` +shard_iops = disk_rate_limit_iops × num_store_paths / num_threads +shard_bytes = disk_rate_limit_mbps × num_store_paths / num_threads +``` + +This assumes shards spread IO uniformly across disks (true on average with +the store-path LUT). Instantaneous skew wastes some ceiling — accepted for +v1; per-(shard, path) buckets are a later refinement if skew shows up in the +gauges. + +**Mechanism.** Two token buckets per shard — `ops` and `bytes` — sharing the +M2 class policy: + +- *Lazy refill, no timers.* The shard event loop already runs continuously; + on each loop iteration (same hook as budget wake today), refill from the + TSC clock: `tokens += rate × (now − last)`, capped at + `rate × rate_limit_burst_ms`. `Shard::ReadTimeMicroseconds()` is already + static and cheap. +- *Debt admission.* `Acquire(cost)` waits until the balance is **positive**, + then subtracts the full cost, allowing the balance to go negative. Long-run + rate equals the refill rate exactly, a single large IO (1MB merged write = + 256 pages of byte-tokens) never deadlocks against a small bucket, and no + submission needs to be split. This also fixes the M1 write-cap quantum + problem measured on 2026-07-21: with count caps, `SubmitMergedWrite` + acquires whole-buffer units, so `max_inflight_write` below + `write_buffer_size / data_page_size` (256 at defaults) cannot bind — the + gauge showed 256 in flight under a cap of 64. Under M4 the byte bucket + charges actual bytes and paces exactly. +- *Class policy: partitioned buckets.* The budget is split by class: + foreground buckets refill at `(100 − rate_bg_ratio)` percent of the + shard rate and are charged only by foreground reads; background buckets + refill at `rate_bg_ratio` percent and are charged by background reads + and all write-path IO. (`rate_bg_ratio` is a separate option from + `bg_read_ratio` because it governs writes too, not only reads.) A + shared-balance variant — foreground draws freely, background capped at + a sub-share — was implemented first and measured worse (2026-07-21): + with writes charged at the device's true accounting granularity, one + merged write's debit drove the shared balance negative and every + foreground read arriving in the next ~0.5 ms/MB waited out the write's + debt (storm p99 1.0 ms shared vs 0.4 ms partitioned). Partitioning + isolates each class's debt at the cost of not lending an idle class's + headroom; **asymmetric borrow-when-idle** (implemented 2026-07-22) + repairs that: foreground only may admit on background's balance, while + background has no waiters and a positive balance, with the debit landing + on background. Read-only throughput recovers to the full configured rate + (243K vs 183K partition-only on the test disk, p99.9 413 µs) while storm + isolation is unchanged (183.5K, ~720 µs, writes pinned to the background + share). Background must never borrow: the symmetric variant was tried + and reverted — a closed-loop foreground's waiting zone empties for + microseconds between completion and resubmission, and in those windows + storm-driven background skimmed the foreground refill wholesale + (measured ~2M borrowed ops/shard per storm; foreground fell 183K → 116K + QPS and p99.9 720 µs → 5.6 ms). Foreground's share is a guarantee + against background and must hold regardless of how idle foreground + momentarily looks. **Accepted consequence (product decision):** because + all write-path IO is background and reverse lending is not implemented, a + pure-write workload (no concurrent foreground reads) runs at only + `rate_bg_ratio` of the device rate — 25% by default — even when the + device is otherwise idle. This is deliberate: the limiter is on by + default and read-tail protection takes priority; pure-write throughput is + no longer a ±3% no-regression guard. Deployments that need full-rate + ingest raise `rate_bg_ratio` (or disable the limiter for a load phase). + Reverse lending (idle foreground donating to background) would remove the + cap but needs genuine idle-hysteresis to stay safe — the instantaneous + "foreground idle?" test that works for foreground borrowing is unsafe in + reverse (a closed-loop foreground looks idle for microseconds between + completions); deferred. + Each refill + wakes each class's zone independently — no cross-class wake coupling, + hence no starvation coupling either. +- *Charging points* are the existing M1 acquire sites: `ReadPage`/`ReadPages` + (ops = pages, bytes = pages × page size), `WritePage` (1 page), + `SubmitMergedWrite` (ops = ceil(len / rate_limit_io_unit) to mirror the + device-command split of large IOs, bytes = len), `fdatasync` (1 op). +- *Waiting/waking* reuse the existing per-class `WaitingZone`s; the wake site + moves from completion (`Release`) to refill (once per loop iteration). + There is no completion-driven release — spent tokens are gone; the refill + is the only credit source. + +**Relation to M1/M2.** The M1/M2 count budgets are **retired** (`IoBudget` +deleted; `max_inflight_read`/`bg_read_ratio` are deprecated no-ops; +`max_inflight_write` reverts to write request-pool sizing). Instantaneous +burst-depth bounding, if a deployment wants it, is the single class-blind +`max_inflight_io` window (off by default; measured inert on the rate-metered +Azure disk). M3 (background write bytes/sec) is subsumed: background write +pacing is the BG class share of the M4 byte bucket — no separate mechanism. + +**Recommended production configuration (validated 2026-07-22): the rate +knobs alone — the count caps are retired.** With the partitioned buckets, +the tuned count caps changed storm p99.9 by nothing measurable (709 vs +722 µs, same throughput), and they were subsequently removed. Configure +`disk_rate_limit_iops` (95% of the measured disk ceiling; default 275K — +the measured Azure v2 local-NVMe ceiling — with rate limiting ON by +default), `rate_bg_ratio` (the one policy choice), and +`rate_limit_io_unit` (the physical write quantum, default 4 KB = one data +page: a 4 KB write costs 1 op, a 1 MB merged write 256; a finer unit +charges writes more ops and paces background harder — not needed at the +tested Azure read-tail target, so left at 4 KB). +`max_inflight_io` (single class-blind in-flight command window) exists as +an off-by-default safety bound; measured inert on Azure — the rate-metered +hypervisor does not penalize instantaneous depth at burst-window +magnitudes, so relocating the queue to user space changes nothing there. + +**New options.** + +``` +uint64_t disk_rate_limit_iops = 275000; // per store_path; 0 = off. ON by + // default: 275K is the measured Azure + // v2 local-NVMe ceiling and a sane + // cloud starting point. Multiple + // store paths are assumed identical + // devices; per-shard budget = + // iops x paths / num_threads. Devices + // faster than ~290K IOPS are capped + // until raised/disabled; measure and + // set 95% of ceiling for precision. +uint64_t disk_rate_limit_mbps = 0; // per store_path; 0 = bytes bucket off +uint32_t rate_limit_burst_ms = 2; // bucket capacity, ms of refill; + // = worst idle-edge latency transient. + // 1/2/4 ms cost no throughput (deep- + // queue A/B 2026-07-22); smaller + // flattens the distribution (median + // up, tail down), larger the reverse. + // 1 ms for tail-first deployments. +uint32_t rate_limit_io_unit = 4096; // write ops quantum: ceil(len/unit) + // ops per write (WriteRateOps, used by + // both WritePage and SubmitMergedWrite). + // Default = one data page: 4KB write = + // 1 op, 1MB merged = 256. Minimum 4KB + // (= default; ValidateOptions rejects + // smaller, the INI loader keeps the + // default on malformed/out-of-range + // values). Coarsen only if the device + // accounts in larger units. +uint32_t rate_bg_ratio = 25; // background share of the rate, percent +``` + +**Calibration.** Measure the disk ceiling once with deep-queue fio (4 jobs × +QD64 exposes the enforced rate directly as the IOPS plateau; the latency +plateau confirms throttling rather than saturation), then set +`disk_rate_limit_iops` to ~90–95% of it. The refill runs on the shard TSC +clock, so that clock's accuracy bounds the limiter's: a calibration bug +(fixed 2026-07-22 — cycles divided by the requested sleep instead of the +measured elapsed time) made the clock ~6% slow and every configured rate +silently deliver 94%; after the fix, delivered rate is within ~1% of +configured, which is also why the 5–10% headroom below the measured +ceiling matters — at 100% the margin is smaller than the ceiling's own +measurement error. On AWS, the documented +instance-store limits (and the NVMe "performance exceeded" counters) give +the number without probing. Leave 5–10% headroom: the hypervisor's own +bucket must never be the binding limiter, or its quantized holds reappear. + +**Validation plan.** + +- Read-only at the ceiling (the 2026-07-21 scenario): sweep + `disk_rate_limit_iops` ±20% around calibrated; expect a *wide* plateau of + good tails (vs. the count cap's knife-edge between 24 and 32 in-flight), + p99.9 within ~2× the true device tail, throughput within 10% of ceiling. +- Mixed storm: foreground p99.9 tracks the read-only number; background + (compaction) absorbs the deficit; write throughput cost reported. +- Cross-check the ops/bytes gauges against `iostat` per phase. + +**Future: adaptive rate.** The provisioned ceiling can be discovered at run +time: on AWS, read the exceeded-counters; elsewhere, detect the throttle +signature (bimodal completion-latency histogram with a fixed ~ms mode) and +walk the rate down until it disappears. Deferred until the static version is +validated in production. + +### New options (per shard) + +> **Superseded (2026-07-22):** the surviving option surface is the M4 +> block below. `max_inflight_read` and `bg_read_ratio` are deprecated +> no-ops, `max_inflight_write` is write request-pool sizing again +> (default back to 32768), and `bg_write_rate_limit` (M3) was never +> shipped — background write pacing is the background share of the M4 +> byte bucket. + +``` +uint32_t max_inflight_read = 64; // configured data pages [DEPRECATED] +uint32_t bg_read_ratio = 25; // percent of max_inflight_read [DEPR.] +uint32_t max_inflight_write = 32768; // configured data pages; effectively + // unbounded by default + // [REVERTED to pool sizing] +uint64_t bg_write_rate_limit = 0; // bytes/sec; 0 = disabled (M3) [never + // shipped; subsumed by M4] +``` + +The WSL interference sweeps (2026-07-03) initially selected a read cap of 32 +and `bg_read_ratio = 25`. Azure NVMe validation (2026-07-11) then showed real +foreground queueing at 32 while 64–128 were indistinguishable, so the shipped +read default is 64 and the policy ratio remains 25%. A later Azure NVMe +acceptance run failed the p99.9 target with write cap 512, so the shipped write +default remains 32768 and smaller caps are opt-in. Real-device calibration (the +QD sweep below) should still re-derive the read cap per device; the ratio is +policy and should transfer. + +### Sizing contract: device knob × policy knob + +The two read-side options deliberately live at different levels: + +- **`max_inflight_read` is device calibration.** Size it from the device's + bandwidth-delay product: `c × max_random_read_IOPS × t_read(unloaded) / + num_threads`, with c ≈ 2–4 for die-imbalance and burst headroom (Azure + local-NVMe calibration 2026-07-11 measured the knee at c ≈ 5–7: budgets + 64–128 for 16 shards on 8 × ~250K-IOPS devices at ~150 µs loaded latency, + indistinguishable within cloud-environment noise; 32 showed genuine + foreground queueing under 128 concurrent readers — the budget must also + cover peak per-shard foreground concurrency, not only the BDP). Below + the BDP the cap costs read throughput; far above it the cap stops + representing the device queue and the ratio contract below degrades. + Validation signal: foreground `read_blocked` should stay ≈ 0 under + representative load — nonzero means undersized. +- **`bg_read_ratio` is policy, and is deliberately a ratio.** A foreground + read's worst-case queueing behind background is `bg_cap / IOPS`; with the + cap BDP-sized this equals `ratio × c × t_read` — the device's IOPS + cancels. Relative tail inflation (tail as a multiple of the device's own + base latency) is therefore a function of the ratio alone: a slower device + serves proportionally slower reads with the **same tail-to-median shape**. + That predictability is the operator-facing contract, it survives hardware + changes without retuning, and it is why the option must not be an + absolute page count. +- Background demand scales with the same device parameters in device-bound + deployments, so the ratio budget and the demand shrink and grow together. + If measured demand persistently exceeds the budget + (`bg_read_blocked_us` ≈ wall time), compaction is being deferred: the + remedy is raising `max_inflight_read` (the device has headroom the + calibration missed), not the ratio — raising the ratio spends the tail + contract. +- **The background slice needs an absolute floor.** Batch writes issue + their own read-modify-write page fetches under the background class, so + a tiny `ratio × cap` throttles ingest itself, not just compaction: + measured at bg_cap = 3 (ratio 10 of cap 32), write throughput halved and + the paced foreground collapsed with it. Until the engine enforces + `bg_cap = max(floor, ratio × cap)` (floor ≈ 8 on tested devices), do not + configure combinations that yield bg_cap below ~8. + +## Interaction with Existing Knobs + +- **`io_queue_size` (SQ ring) stays.** With both caps ≪ 4096, tasks + should essentially never block in `GetSQE`; the ring cap becomes a sanity + bound rather than a throttle. +- **`max_write_batch_pages` throttle retires.** *(Implemented, plan commit + 4.)* The `inflight_io_ >= cap → WaitWrite()` branch in + `WriteTask::WritePage` is subsumed by M1/M2 and is strictly worse (it + drains to zero and restarts, producing a sawtooth; budget acquisition + keeps a steady level). Keep: the CPU yields between page builds + (`YieldToLowPQ()` / `MaybeYield()` — the #455 time-budgeted discipline is + orthogonal and stays), and the terminal `WaitWrite()` before + `UpdateMeta`/`SyncData` (error collection and durability ordering). + One consequence surfaced by tests: with `enable_data_page_cache`, a write + task's write-promotion pins on cached pages are now bounded by + `max_inflight_write` rather than by the per-task drain, so pathologically + small buffer pools must account for in-flight-write pins (production + pools dwarf the ≤ max_inflight_write pages of pins; only tests noticed). +- **`max_inflight_write` is redefined, not retired.** It is the M1 write cap + (configured data-page units). The default remains 32768, effectively + unbounded; 512 is an explicit opt-in value until the release acceptance + target is met. + In-flight pages are bounded by `max(cap, one request's cost)` because one + oversized request may run alone to guarantee progress. `WriteReqPool` stays + numerically sized to the option, but counts request objects rather than page + units; it is therefore a conservative allocation bound, not the QoS bound + itself. Note this is a behavioral change for deployments that set the old + option explicitly. +- **`max_write_concurrency` is no longer an IO-QoS knob.** Note it is + enforced **per shard** (each shard's `TaskManager` counts its own active + write tasks against the store-wide option value), so the device-wide task + bound is `max_write_concurrency × num_threads`. It remains as a + cross-partition admission/memory bound (coroutine stacks, 1MB write + buffers, CoW roots with retained mapping snapshots — per-partition writes + are already serialized by `PendingWriteQueue`). Local-mode guidance: + a small constant (k = 2–4). Rationale: + - k = 1 forfeits intra-shard pipelining: each batch's serial IO tail + (final drain + fdatasync + manifest append) becomes dead time for that + shard's queued partitions. k = 2 recovers most of it. + - With 4–8 shards per device, device-level parallelism, utilization, and + stream mixing are set by shard count, not k; k is purely an intra-shard + pipelining knob. Per-shard k also provides skew robustness when write + load concentrates on few shards. + - Small k additionally staggers fdatasync phases, avoiding fsync bursts. + - Cloud mode keeps `max_cloud_concurrency`-scale values: the terminal + stall is an upload (tens of ms), so higher k is required to keep the + shard's write pipeline busy. + +## Multi-Shard Considerations + +All budgets are per shard, but the device is shared by all shards +(typically 4–8 per NVMe device): + +- Size per-shard budgets as device budget / num_threads. Static division is + safe (aggregate background pressure is bounded regardless of which shards + are busy) but conservative: a lone compacting shard gets 1/S of the + background budget. Acceptable initially. +- Follow-up: a shared global token bucket (atomic counter) for the + background budget. Cross-shard wake-up is handled by each shard's + `WorkLoop` re-checking the bucket once per round and waking its local + waiters — no cross-thread coroutine resumption needed. +- Cloud mode: NIC bandwidth is shared between background uploads and + foreground cache-miss downloads; the same FG/BG discipline should + eventually apply to cloud slots. Whole-file upload, prewarm/download, and + snapshot disk IO remains exempt from the page-IO budget, including + `ReadFile`, `ReadFilePrefix`, and `WriteSnapshot` bulk paths. + +## Non-Goals + +- **Segment IO (zero-copy large values) is not budgeted.** Segments exist to + serve very large values, a workload where throughput matters rather than + tail latency and fairness, so FG/BG isolation is not a goal there. + Segment IO is counted against neither `max_inflight_read` nor + `max_inflight_write`, and is not classified; + segment compaction keeps its existing bounds (`max_segments_batch` + in-flight segments per batch via the shared `GlobalRegisteredMemory` pool, + `segment_compact_yield_every`). Revisit only if large-value traffic is + ever mixed with latency-sensitive point reads on the same store. +- No `sqe->ioprio` reliance: only honored by mq-deadline/BFQ schedulers + (NVMe defaults to `none`). May be added as a one-line supplement, never as + the mechanism. +- No deferred BG staging queue (strict priority instead of a ratio): more + complexity; revisit only if M1–M3 prove insufficient. +- No cross-shard coordination in the first iteration. + +## Observability + +Export per-shard counters from day one; tuning must be measurement-driven: + +- Current and high-watermark pages for total reads, the BG-read subset, and + writes; foreground read usage is total minus BG. +- Cumulative blocked time and counts per admission class: `read_` is foreground + waits only, `bg_read_` is background waits, and `write_` is all write waits. +- Background write bytes/sec (actual, vs. limit when M3 lands). +- fdatasync count and cumulative batch wall time. + +## Evaluation Plan + +0. **Re-baseline on top of #455.** The interference experiments that + motivated this design predate the time-budgeted yield and GC chunking. + Re-run the read-vs-compaction benchmark first to quantify how much + foreground tail latency remains attributable to device-side queueing + (this design's target) vs. worker-thread stalls (already fixed). The + remaining gap sets the success criterion for M1+M2. +1. **Device calibration sweep** (per target device, preconditioned drive): + fixed 4KB random-read load at target QD, step up sequential write MB/s, + plot read p99/p99.9 vs. write rate. The knee is the background budget. + Expect a near-linear tail-latency curve for the sequential-write pattern. + Run long enough to exhaust any SLC cache so consumer-drive folding + behavior does not confound results. +2. **Isolate fdatasync impact**: repeat the interference experiment with + sync counts instrumented; if fsync stalls dominate, sync spacing/batching + matters more than page-IO budgets and no in-flight cap will fix it. +3. **Verify TRIM**: confirm the filesystem issues discards for GC-deleted + files (`discard` mount option or periodic `fstrim`); otherwise the + WAF ≈ 1 assumption silently breaks on aged drives. +4. **Historical staged rollout** (M1/M2 and throttle retirement are complete): + - Land M1+M2 with existing throttles left in place (loosened). + - Re-run the read-vs-compaction benchmark; confirm the new mechanism + alone protects foreground p99. + - Remove the `max_write_batch_pages` throttle, but keep + `max_inflight_write` effectively unbounded by default until a calibrated + QoS value meets the release acceptance target. + - Sweep `max_write_concurrency` ∈ {1, 2, 4, 8} in local mode; pick the + throughput knee (expected at 2–4). + - Evaluate M3 against the calibration curve; enable if M1+M2 leave a + sustained-write tail-latency gap. diff --git a/docs/design/io_qos_impl_plan.md b/docs/design/io_qos_impl_plan.md new file mode 100644 index 00000000..ffeab567 --- /dev/null +++ b/docs/design/io_qos_impl_plan.md @@ -0,0 +1,366 @@ +# IO QoS: Implementation and Testing Plan + +Companion to `io_qos.md` (the design). This document maps M1/M2 (and the M3 +follow-up) onto concrete code changes, commit sequence, and tests. File and +symbol references are as of `main` @ 8f5f699. + +## Commit sequence + +Five commits, each independently landable and revertible. Docs +(`docs/architecture/04-execution-model.md`, `07-io-stack.md`, +`08-data-lifecycle.md`) are updated in the same commit as the behavior they +describe (CLAUDE.md requirement). + +| # | Content | Risk | +|---|---|---| +| 1 | M1: budget state, acquire/release plumbing, options, stats getter | Core; behavior-neutral at default caps until tightened | +| 2 | M2: `IsBackground()`, BG read sub-budget, wake policy | Small delta on top of 1 | +| 3 | Interference benchmark + fio calibration script | Test-only | +| 4 | Retire `max_write_batch_pages`; keep write QoS opt-in until acceptance passes | Behavioral; gated on benchmark results | +| 5 | M3 rate limiter (deferred; only if step-4 benchmarks show a sustained-write gap) | Optional | + +## Commit 1 — M1: in-flight page-IO budgets + +> **Status: implemented** (2026-07-02). Deviations from the plan below: +> `IoQosStats` includes fdatasync counters; the oversized-request escape +> (`Acquire` admits a request costlier than the cap once the budget drains) +> was added so small configured caps cannot deadlock against ~1MB merged +> writes; tests use `REQUIRE` (glog's `CHECK` macro shadows Catch2's). + +### State (in `IouringMgr`, per shard; `MemStoreMgr` untouched) + +```cpp +// include/async_io_manager.h +class IoBudget +{ +public: + void Acquire(uint32_t cost); // blocks ThdTask() on waiting_ (FIFO) + void Release(uint32_t cost); // decrement + WakeN; called by PollComplete + // stats: current_, high_watermark_, total_blocked_count_, ... +private: + uint32_t inflight_{0}; + uint32_t cap_{0}; // 0 = disabled (Acquire is a no-op) + WaitingZone waiting_; +}; + +IoBudget read_budget_; // cap = max_inflight_read +IoBudget write_budget_; // cap = max_inflight_write (redefined) +``` + +Follow the trailing-underscore member convention. `Acquire` loops +`while (inflight_ + cost > cap_) waiting_.Wait(ThdTask());` — the counter +holds only admitted, not-yet-completed IO (see design doc pseudocode). + +### Release-side classification: new `UserDataType`s + +`PollComplete` must know each CQE's class and cost. `EncodeUserData` packs +the type into the low 8 bits (`async_io_manager.cpp:1519`), so there is +ample room. Current types: `KvTask`, `BaseReq`, `WriteReq`, +`MergedWriteReq`. Add: + +- `KvTaskPageRead` — single-page read issued via the `KvTask` path + (`IouringMgr::ReadPage`). Cost 1 configured data page. Handled identically to + `KvTask` in + `PollComplete`, plus `read_budget_.Release(1)`. +- `BaseReqPageRead` — per-page read in a batch (`IouringMgr::ReadPages`). + Handled identically to `BaseReq`, plus `read_budget_.Release(1)`. + +Metadata ops keep `KvTask`/`BaseReq` and are never charged. Write costs are +already derivable in configured data-page units: `WriteReq` = 1; +`MergedWriteReq` = `bytes_ / data_page_size` (round up; assert page alignment). + +FG/BG at release time (needed by commit 2) comes from the task pointer every +req type already carries: `task->IsBackground()`. + +### Acquire points and ordering + +Rule (from the design): acquire the budget **last**, immediately before +`GetSQE`, after every other blocking acquisition (FD open, req-pool alloc, +write-buffer alloc). This gives a consistent global order: FD/mutex → +pools/buffers → budget → SQE, and guarantees a task never holds budget while +blocked on another pool. + +To be precise about the two phases: *waiting inside* `Acquire` is expected — +the task parks on the budget's `WaitingZone` holding no budget (the counter +is only incremented on admission), and `PollComplete` wakes it regardless of +scheduling. The "no yield points" rule applies *after admission*: between +incrementing the counter and submitting the SQE there must be no voluntary +yield (`YieldToLowPQ`/`MaybeYield`) and no blocking on any resource whose +release could depend on this task. `GetSQE`'s ring-full wait is the one +tolerated block in that window: benign (slots are freed by kernel progress, +never by the blocked task) and unreachable in practice with caps ≪ +`io_queue_size`. + +| Entry point | Change | +|---|---| +| `IouringMgr::ReadPage` (`async_io_manager.cpp` ~484) | `read_budget_.Acquire(1)` at the top of the `read_page` lambda's retry loop, before `GetSQE`. Release is per-CQE, so each retry re-acquires. | +| `IouringMgr::ReadPages` (~565) | Acquire **per page** inside `send_req`, not per batch. Rationale: a 128-page compaction batch must not deadlock against a BG sub-budget of 64 (commit 2); per-page acquisition lets the task block mid-batch while already-submitted pages complete. This supersedes the "atomic batch acquire" idea discussed during design review. | +| `IouringMgr::WritePage` (~742) | `write_budget_.Acquire(1)` after `write_req_pool_->Alloc`, before `GetSQE`. | +| `IouringMgr::SubmitMergedWrite` (~777) | `write_budget_.Acquire(ceil(bytes / data_page_size))` after `merged_write_req_pool_->Alloc`, before `GetSQE`. | + +Exempt (unchanged): all metadata ops, manifest IO, `ReadFile` / +`ReadFilePrefix` / `WriteSnapshot` bulk paths, `Fdatasync` (instrumented only), +`ReadSegments` / `WriteSegments` (out of scope per design Non-Goals). + +### Release point + +In `PollComplete` (`async_io_manager.cpp` ~2010), per CQE, after the +existing per-type handling: `Release(cost)` on the matching budget. Wake +policy in commit 1 is a plain `WakeN` on the released budget's zone. + +### Options and validation + +- `kv_options.h`: add `max_inflight_read` (initially 256; shipped default 64); + redefine `max_inflight_write` in configured data-page units (keep old default + 32768 through release acceptance; lower caps remain opt-in). Add INI parsing + in `kv_options.cpp` and equality-operator entries. +- `eloq_store.cpp` option validation: `max_inflight_write != 0` already + enforced; nothing else hard-fails. `max_inflight_read == 0` disables the + read budget (documented). +- `WriteReqPool` stays sized by `max_inflight_write` (unchanged line in + `IouringMgr` ctor) as a conservative request-object allocation bound. QoS + counts configured page units, and a merged request may consume multiple + units. + +### Stats + +`struct IoQosStats` (per budget: current, high-watermark, blocked count, +cumulative blocked µs) + `IouringMgr::GetIoQosStats()`. `read_` blocked fields +count foreground waits only, `bg_read_` counts background waits, and `write_` +counts all write waits. The `ELOQSTORE_WITH_TXSERVICE` meter registers gauges +for current total-read, BG-read, and write usage behind the existing +`EnableMetrics()` guard; full stats remain reachable through the store's shard +accessors. Add an fdatasync counter + latency accumulator in +`SyncFiles`/`FdatasyncFiles` in the same commit (evaluation step 2 needs +it). + +### Shutdown / abort audit (do during code review of commit 1) + +- Tasks blocked in a budget `WaitingZone` are `TaskStatus::Blocked`, so + `TaskManager::NumActive() > 0` keeps `Shard::WorkLoop` spinning and + `PollComplete` continues releasing budget — no shutdown hang. Same + property as the existing `WriteReqPool::waiting_`. +- `AbortWrite` calls `WaitIo()` first; all of the aborting task's CQEs drain + through `PollComplete`, releasing their budget. No manual cleanup needed. +- Kill-point paths (`TEST_KILL_POINT_WEIGHT("WritePage", ...)`) fire before + budget acquisition — keep it that way. + +## Commit 2 — M2: FG/BG read sub-budget + +> **Status: implemented** (2026-07-02). As planned, with two refinements: +> (a) the sub-budget lives inside `IoBudget` (`SetBgCap` + `Acquire/Release` +> taking a `background` flag) rather than as a separate class, so the write +> budget reuses the same type with no sub-budget configured; per-class +> oversized-request escape and FIFO zones as described; stats gained a +> `bg_read_` slice and a BG-read inflight gauge. (b) Waking FG first was found +> to starve background under sustained foreground saturation. Demand is now +> tracked from first wait through admission, so the unused BG entitlement stays +> reserved across wake-to-admit; release wakes BG first and always wakes FG. + +- `KvTask::IsBackground()` in `tasks/task.h`: `BatchWrite`, + `BackgroundWrite`, `EvictFile`, `Prewarm` → true; explicitly not + `ReadOnly()`. +- Extend the read budget: `bg_inflight_` counter and a second `WaitingZone` + (`bg_waiting_`). Acquire for a BG task additionally checks + `bg_inflight_ + cost > bg_cap_`. Release decrements both counters when + the completing request's `task->IsBackground()`. +- Wake policy: on read-budget release, wake BG waiters while + `bg_inflight_ < bg_cap_`, and always wake FG as well. Spurious wakes are safe + (acquire re-checks in its loop). +- Option `bg_read_ratio = 25` (percent, clamp 1–100); + `bg_cap_ = max_inflight_read * ratio / 100`, min 1 page... but see next + line: `bg_cap_` must be ≥ 1 and the per-page acquisition from commit 1 + guarantees any batch size makes progress through an arbitrarily small + sub-budget. + +## Commit 3 — benchmark and calibration tooling + +> **Status: implemented** (2026-07-02). Delivered as a standalone binary +> `benchmark/interference_bench.cpp` (+ `opts_interference.ini`) rather than +> extending `eloq_store_bm`, and `scripts/io_calibration_sweep.sh`. Two +> deviations from the sketch below: (a) the storm is a **rotating strided +> partial overwrite** (3-of-5 keys, ~3000B one-KV-per-page values, shifted +> each round) — a bulk overwrite leaves files fully dead and compaction +> generates no read traffic at all; (b) latency percentiles are exact +> (raw-sample sort at phase end), not sliding-window. The report includes +> per-shard IoQosStats deltas (BG-read watermark/blocked, fdatasync +> count/latency), giving the sweep direct visibility into whether the +> budgets engaged. Smoke-verified: storm lifts read p99 ~2.3× at toy scale, +> bg_read watermark pins at the configured sub-budget cap. + +- **Interference benchmark**: mixed scenario: steady point-read load at + fixed QD on N partitions while a write/compaction storm runs. Report read + p50/p99/p99.9 and BG write MB/s. Accept knob overrides via the existing + `opts_*.ini` mechanism so sweeps are scriptable. +- **fio calibration script** (`scripts/io_calibration_sweep.sh`): fixed + 4KB randread job + stepped sequential-write job against the target + device; emits the p99-vs-write-MB/s curve (design evaluation step 1). + Document drive preconditioning and SLC-exhaustion runtime in the script + header. +- Run design evaluation step 0 (re-baseline on top of #455) with this + benchmark before tightening any default. + +## Commit 4 — retire superseded throttles (gated on benchmarks) + +> **Status: implemented** (2026-07-03) except for default tightening. The +> throttle retirement shipped in the branch, but the later Azure NVMe +> acceptance run rejected making the calibrated write cap the default. +> Details: `WritePage` throttle branch removed (budget admission comment +> left in its place); `max_write_batch_pages` deprecated — parsed with a +> LOG(WARNING), validation removed so all values are accepted, field documented +> as no-effect; +> The WSL sweep initially selected `max_inflight_write = 512` from +> {512, 2048, 32768} (512 bound marginally — hwm pinned, +> 6–26 blocks/30s — at equal-or-best write throughput and read tails; +> natural demand ceiling was 2048 = 8 concurrent 1MB merged writes). +> The later Azure NVMe acceptance run did not justify making that value the +> default, so the final default remains 32768 and 512 is opt-in. One behavioral +> consequence of selecting a smaller value: with +> enable_data_page_cache, write-promotion pins are now bounded by +> max_inflight_write instead of the per-task drain; the data_page_cache OOM +> test was recalibrated to `max_inflight_write = 1` (tiny 2-slot pool). +> The read default moved from 256 to 32 in the initial WSL campaign, then to +> the shipped value 64 after Azure NVMe validation; bg_read_ratio stays 25. + +The implemented change was gated on the interference benchmark confirming +that M1+M2 alone protect foreground p99 (design evaluation step 4): + +- Removed the `inflight_io_ >= max_write_batch_pages → WaitWrite()` branch in + `WriteTask::WritePage` (`write_task.cpp` ~304). Keep the CPU yields and + the terminal `WaitWrite()`. +- Kept `max_inflight_write` at 32768 by default; calibrated lower values such + as 512 are explicit opt-ins. Deployments setting the option must interpret + it under its new page-budget semantics. +- Deprecated `max_write_batch_pages` in `kv_options.h` (keep parsing, + ignore, log a warning) for one release before deletion. + +## Commit 5 — M3 rate limiter (deferred) + +Sketch only; build if commit-4 benchmarks show sustained-compaction tail +degradation that inflight caps don't remove: token bucket (bytes) in +`IouringMgr`, refilled from the shard `WorkLoop` clock each round +(`ReadTimeMicroseconds` is already sampled per round), debited in +`WritePage`/`SubmitMergedWrite` when `task->IsBackground()`, bucket capacity +≤ 4 refill intervals, `bg_write_rate_limit = 0` disables. + +## Testing plan + +> **Status:** unit-test items 1–6 and the later regression cases are implemented +> in `tests/io_qos.cpp` (16 cases). Notes: item 4's "two concurrent merged +> writes" requires multiple partitions' write tasks — a single task's +> flushes don't overlap (it yields per page while filling the next buffer); +> item 5 uses the read-only-directory injection (kill points SIGTERM the +> process — they are crash-recovery tooling, not error injection); item 6 +> verifies Stop() drains tasks queued behind a 1-page budget. Regression: +> full suite run (memlock binaries via systemd-run on WSL); db_stress with +> tiny caps (read 4 / bg 1 / write 8, so append merged writes exercise the +> oversized-admission path every flush) run ≥100 iterations across +> non-append + append modes — db_stress needs `active_width <= max_key` +> or its sliding window indexes out of bounds (harness limitation, debug +> builds assert). ASAN: two modes exist — `WITH_ASAN=ON` (fully sound, +> including coroutine stacks) hard-requires a locally built +> `libboost_context-asan` (Boost.Context compiled with BOOST_USE_ASAN + +> ucontext; not a distro package); alternatively, injecting +> `-fsanitize=address` via CMAKE_CXX_FLAGS with WITH_ASAN=OFF links stock +> boost and gives full **heap** coverage (UAF/overflow/leaks — what the +> budget/pool plumbing needs) with degraded stack coverage inside coroutine +> frames. The QoS suites were run under the flag-injection mode on this +> box. +> +> Results (2026-07-03, WSL box): full suite 32/32 binaries green (memlock +> suites via systemd-run; persist's one failure was cross-load contention, +> 3/3 clean serially). db_stress: 120/120 iterations pass (60 non-append + +> 60 append, tiny caps). One real commit-4 fallout found by the sweep: +> data_page_cache "OOM retry under concurrent writes" became ~1-in-5 flaky +> — it runs append mode with a pool too small for a write-buffer pool, so +> writes take the non-append path where the removed per-task drain used to +> bound write-promotion pins; fixed by bounding pins via +> `max_inflight_write = 32` in the test (12/12 pass after). Same class of +> consequence as the other recalibrated OOM test; production pools are +> orders of magnitude above the ≤ max_inflight_write pin bound. + +### Unit tests (new `tests/io_qos.cpp`, Catch2, wired into `tests/CMakeLists.txt`) + +Use `tests/common.h` presets; add a `qos_opts` preset with deliberately tiny +caps to make blocking paths hot: + +1. **Accounting invariants**: mixed read/write workload on `default_opts` + + `append_opts` variants with `max_inflight_read = 4`, + `max_inflight_write = 8`; after quiesce, assert both budgets' current + == 0 and high-watermark ≤ cap. Repeat with caps disabled (0) — stats + stay zero, results identical. +2. **Oversized batch vs. tiny cap**: values with >128 overflow pages (forces + `GetOverflowValue`'s 128-page `ReadPages` batches) under + `max_inflight_read = 4` — completes, no deadlock (validates per-page + acquisition). +3. **BG sub-budget**: issue a `CompactRequest` concurrently with a stream of + point reads under small caps; assert BG high-watermark ≤ `bg_cap_`, + foreground reads complete, and compaction finishes (no starvation in + either direction). +4. **Merged-write cost**: append mode with `max_inflight_write = 256` and + 1MB write buffers — two merged writes cannot be in flight together + (watermark ≤ 256); with 512 they can. +5. **Error paths**: kill-point injection on `WritePage` (existing + `TEST_KILL_POINT_WEIGHT`) and a failing write (read-only dir trick or + fault injection as in `large_value_fault_injection.cpp`) — budget + returns to zero after `AbortWrite`. +6. **Shutdown while blocked**: stop the store while tasks are queued behind + a 1-page budget; store drains and stops cleanly. + +### Regression and stress + +- Full `ctest --test-dir build/tests/` with default options (must be + behavior-neutral before commit 4) and again with the tiny-cap `qos_opts` + exported via an env override if practical. +- `db_stress` run with tiny caps in both append and non-append modes; + repeat-run (≥100 iterations) to shake out wake/ordering races — budget + bugs manifest as rare hangs, same failure mode as the historical + free-list ABA issue, so favor iteration count over single-run length. +- ASAN build (`build-asan/`) for the new plumbing. +- On WSL2 dev machines, launch memlock-sensitive suites via + `systemd-run --user --pipe --wait --property=LimitMEMLOCK=2G` (ulimit is + unreliable there); do not shrink test parameters to dodge it. + +### Performance acceptance (per target device, after commit 3) + +- **User-confirmed product target**: read p99.9 below 10 ms during concurrent + write, compaction, and GC. +- **Measured status: FAIL / not release-ready.** + - In the 2026-07-16 same-binary campaign, median read p99.9 was + 19.779 ms with read QoS disabled and 18.711 ms with read QoS enabled. + Both conditions retained the 512-page write budget, so that campaign did + not isolate its effect. It ran at + `c625004a32f474f446ba8adeba2d2d68f93dcee7` on `/dev/nvme1n1` + (`Microsoft NVMe Direct Disk v2`). + - The 2026-07-17 three-condition follow-up used three balanced-order, + fresh-store repetitions at + `42b7f94084f196264615a77ccb20a35c284f5d12`. Median logical write + throughput was 19,473 key-ops/s with write cap 32768, 19,337 with write + cap 512 (-0.7%), and 19,693 with the full 64/25%/512 policy. The 512 cap + reached its high-water mark and recorded a median 53 write blocks, but was + not sustained-throughput-discriminating. Median read p99.9 was + 17.111/16.268/15.751 ms respectively: the full policy improved 7.9% versus + 32768 but still missed the 10 ms target by 57.5%. All nine runs had + concurrent compaction/GC and passed their validity gates. + - The tested configurations set their caps explicitly, so the subsequent + default restoration does not change the exercised admission + configuration. The retained run metadata did not record + `ELOQ_IO_STATS`; the later timing cleanup is diagnostic-only, but this + campaign was not repeated at that later commit. +- **Outstanding no-regression guards**: pure-read throughput and pure-write + throughput within noise (±3%) of the pre-QoS baseline at default caps — the + hot-path cost is two counter checks per IO, so any regression indicates a + wake storm or false blocking. +- **Outstanding sweeps**: `max_inflight_read` ∈ {64, 128, 256, 512}, + `bg_read_ratio` ∈ + {10, 25, 50}, `max_inflight_write` ∈ {256, 512, 1024}, + `max_write_concurrency` ∈ {1, 2, 4, 8}; record read tails + BG + throughput; pick defaults at the knees. + +### Documentation updates (same commits as code) + +- `07-io-stack.md`: budgets, new UserDataTypes, acquire/release points. +- `04-execution-model.md`: budget blocking as a new task wait state, + interaction with `WaitingZone`/`MaybeYield`. +- `08-data-lifecycle.md`: compaction/GC now run under the BG read budget. +- `io_qos.md`: mark mechanisms as implemented as they land. diff --git a/include/async_io_manager.h b/include/async_io_manager.h index 19a00292..1ce7e8ff 100644 --- a/include/async_io_manager.h +++ b/include/async_io_manager.h @@ -54,6 +54,153 @@ class ManifestFile virtual KvError SkipPadding(size_t n) = 0; }; +/** + * @brief Per-shard device rate budget (docs/design/io_qos.md M4). + * + * Two token buckets — device operations and bytes — refilled lazily from + * the shard clock once per event-loop iteration (RefillAndWake, called + * from IouringMgr::Submit) and spent at the same entry points as the page + * IO paths. Admission uses debt semantics: a task waits until the + * balances are positive, then subtracts its full cost, letting a balance + * go negative. The long-run rate therefore equals the refill rate + * exactly, and a single IO costlier than the bucket capacity (a 1MB + * merged write against a small bytes bucket) admits without deadlock — + * it just drives the balance into debt that subsequent refills pay off + * before the next admission. + * + * The budget is PARTITIONED by class, not shared: foreground buckets + * refill at (100 - rate_bg_ratio) percent of the rate and are charged + * only by foreground IO; background buckets refill at rate_bg_ratio + * percent and are charged by background IO (background reads and all + * write-path IO). A shared balance was tried first and rejected: one + * large background debit (a merged write charged at the device's + * accounting granularity) drove the common balance negative and stalled + * every foreground read behind the write's debt for ~0.5 ms per MB + * (measured 2026-07-21, storm p99 1.0 ms shared vs 0.4 ms partitioned). + * Partitioning alone would waste idle-background headroom; ASYMMETRIC + * borrowing repairs that: FOREGROUND ONLY may admit on background's + * balance, and only while background has no waiters and a positive + * balance; the debit lands on background (see CanAdmit). With background + * idle, foreground runs at the full configured rate; the moment + * background demand appears, lending stops and background resumes at + * worst one IO-cost below zero. Background never borrows — foreground's + * share is a guarantee against background, and symmetric borrowing + * measurably collapsed that guarantee (see CanAdmit). + * There is deliberately no Release(): a rate budget meters issuance per + * second, not occupancy. A token is consumed by sending the IO, and how + * fast the device completes it is irrelevant — refunding on completion + * would collapse the rate limit into a mere concurrency ceiling. (The + * occupancy role belongs to the max_inflight_io window, which does pair + * AcquireIoWindow with a completion-time ReleaseIoWindow.) + * Wake-ups are refill-driven, never completion-driven: spent tokens are + * gone, time is the only credit source, so waiter progress depends only + * on the shard loop running. Each refill runs peek-and-grant per class + * (GrantWaiters): the waker charges the FIFO head's recorded cost on its + * behalf before waking it, so wake counts are exact even with + * heterogeneous costs and woken tasks never re-queue. + * + * Balances are stored scaled by kScale (1 token = kScale units) so refill + * arithmetic (rate/sec x elapsed microseconds) stays in integers. + */ +class RateBudget +{ +public: + void SetRates(uint64_t ops_per_sec, + uint64_t bytes_per_sec, + uint32_t burst_ms, + uint32_t bg_ratio_pct); + bool Enabled() const + { + return fg_ops_rate_ != 0 || fg_bytes_rate_ != 0 || bg_ops_rate_ != 0 || + bg_bytes_rate_ != 0; + } + void Acquire(uint32_t ops, uint64_t bytes, bool background); + void RefillAndWake(uint64_t now_us); + IoQosStats::Rate Stats() const + { + return {blocked_count_.load(std::memory_order_relaxed), + blocked_us_.load(std::memory_order_relaxed), + admitted_ops_.load(std::memory_order_relaxed), + admitted_bytes_.load(std::memory_order_relaxed), + borrowed_ops_.load(std::memory_order_relaxed)}; + } + IoQosStats::Rate BgStats() const + { + return {bg_blocked_count_.load(std::memory_order_relaxed), + bg_blocked_us_.load(std::memory_order_relaxed), + bg_admitted_ops_.load(std::memory_order_relaxed), + bg_admitted_bytes_.load(std::memory_order_relaxed), + bg_borrowed_ops_.load(std::memory_order_relaxed)}; + } + +private: + static constexpr int64_t kScale = 1'000'000; + + /** + * @brief Whether the class's own balances are all positive. + */ + bool Positive(bool background) const; + /** + * @brief Admission test with asymmetric borrow-when-idle. + * + * A class is admissible on its own positive balances; foreground + * additionally on background's — while background has NO waiters (the + * instant its demand appears, lending stops) and a positive balance + * (surplus is lent, debt is never transferred). The borrow debit + * lands on background's buckets, so its refill repays the loan and it + * restarts at worst one IO-cost below zero. Background never borrows. + * + * @param background The class requesting admission. + * @return True if an acquisition of this class may proceed now. + */ + bool CanAdmit(bool background) const; + /** + * @brief Charge @p ops / @p bytes to the granting balances (own, or + * the lender's when borrowing) and update the class's counters. Debt + * semantics: balances may go negative. + */ + void Charge(uint32_t ops, uint64_t bytes, bool background); + /** + * @brief Peek-and-grant admission for the class's FIFO: while the + * class is admissible, charge the head waiter's recorded cost + * (KvTask::rate_wait_ops_/bytes_) on its behalf, then wake it. Exact + * wake counts for heterogeneous costs; the woken task's Acquire tail + * does nothing — this is the only waker of the rate zones, so a wake + * means the charge already happened. + */ + void GrantWaiters(bool background); + + // Per-class token rates per second; 0 disables that bucket. + uint64_t fg_ops_rate_{0}; + uint64_t fg_bytes_rate_{0}; + uint64_t bg_ops_rate_{0}; + uint64_t bg_bytes_rate_{0}; + uint64_t burst_us_{0}; // bucket capacity, microseconds of refill + uint64_t last_refill_us_{0}; // 0 = first refill pending + // Scaled balances (1 token = kScale units); may go negative (debt). + int64_t fg_ops_bal_{0}; + int64_t fg_bytes_bal_{0}; + int64_t bg_ops_bal_{0}; + int64_t bg_bytes_bal_{0}; + // Observability only; admission reads nothing but rates and balances. + // Atomic (relaxed) because IoQosStats is sampled from other threads + // while the shard runs (see the concurrent-sampling test); the shard + // thread is the only writer. + std::atomic blocked_count_{0}; + std::atomic blocked_us_{0}; + std::atomic admitted_ops_{0}; + std::atomic admitted_bytes_{0}; + std::atomic bg_blocked_count_{0}; + std::atomic bg_blocked_us_{0}; + std::atomic bg_admitted_ops_{0}; + std::atomic bg_admitted_bytes_{0}; + // Ops admitted from the other class's surplus (borrow-when-idle). + std::atomic borrowed_ops_{0}; + std::atomic bg_borrowed_ops_{0}; + WaitingZone waiting_; // foreground waiters + WaitingZone bg_waiting_; // background waiters +}; + using ManifestFilePtr = std::unique_ptr; // TODO(zhanghao): consider using inheritance instead of variant @@ -98,8 +245,27 @@ class AsyncIoManager { return store_stopping_.load(std::memory_order_acquire); } + /** + * @brief Top-of-round kernel entry: refill the device rate budget (its + * only wake source) and drive io_uring submission / task_work. + */ virtual void Submit() = 0; virtual void PollComplete() = 0; + /** + * @brief End-of-round flush: push SQEs prepared by ExecuteReadyTasks to + * the kernel in the same round instead of leaving them for the next + * round's Submit. + * + * Splitting this out of Submit keeps the rate-budget refill at exactly + * one per round while removing the round-boundary delay between + * preparing an IO and issuing it — which in module mode (an embedding + * runtime driving Process/HasTask) is a full external scheduling + * quantum of dead device time per IO hop. No-op when the round prepared + * nothing. + */ + virtual void FlushSubmit() + { + } virtual bool NeedPrewarm() const { return false; @@ -196,6 +362,16 @@ class AsyncIoManager return 0; } + /** + * @brief Per-shard IO QoS statistics (in-flight page-IO budgets and + * fdatasync accounting). Default zeros for managers without budgets + * (MemStoreMgr). See docs/design/io_qos.md. + */ + virtual IoQosStats GetIoQosStats() const + { + return {}; + } + /** * @brief Read K segments into pre-registered buffers. * @@ -541,6 +717,19 @@ class IouringMgr : public AsyncIoManager ~IouringMgr() override; KvError Init(Shard *shard) override; void Submit() override; + void FlushSubmit() override; + /** + * @brief A shard must not idle-block while this ring has prepared or + * submitted-but-unreaped IO: CQE delivery under DEFER_TASKRUN + * requires this thread to keep entering the kernel. The base class + * returns true unconditionally, which let shards sleep up to the + * 100ms request-wait timeout with CQEs pending (observed during + * prewarm, whose IO is not owned by an active task). + */ + bool IsIdle() override + { + return inflight_ios_ == 0; + } void PollComplete() override; char *AcquireWriteBuffer(uint16_t &buf_index) override; void ReleaseWriteBuffer(char *ptr, uint16_t buf_index) override; @@ -768,7 +957,20 @@ class IouringMgr : public AsyncIoManager KvTask, BaseReq, WriteReq, - MergedWriteReq + MergedWriteReq, + // Data-page reads charged against the read IO budget (see + // docs/design/io_qos.md). Named + PageRead: the + // payload and completion handling are identical to the plain + // KvTask / BaseReq types, plus a read-budget release. The distinct + // types exist so PollComplete can tell budgeted data-page reads + // apart from metadata ops that share the plain payload types. + KvTaskPageRead, // ReadPage (single page), payload = KvTask* + BaseReqPageRead, // ReadPages (one page of a batch), payload = BaseReq* + // Batch fsyncs charged against the io window (FdatasyncFiles), + // payload = BaseReq*. Distinct from BaseReq so PollComplete can + // release the window command without touching the metadata ops + // (open, close, unlink, ...) that stay window-exempt. + BaseReqFsync }; struct BaseReq @@ -1032,6 +1234,19 @@ class IouringMgr : public AsyncIoManager WaitingZone waiting_sqe_; uint32_t prepared_sqe_{0}; + RateBudget rate_budget_; + // Single class-blind in-flight device-command window (max_inflight_io; + // see AcquireIoWindow). 0 cap = disabled. The shard thread is the only + // writer; atomics (relaxed) allow cross-thread stats sampling. + uint32_t io_window_cap_{0}; + std::atomic io_window_inflight_{0}; + std::atomic io_window_hwm_{0}; + std::atomic io_window_blocked_{0}; + WaitingZone io_window_waiting_; + // Write-path fdatasync instrumentation (FdatasyncFiles batches). + std::atomic fdatasync_count_{0}; + std::atomic fdatasync_us_{0}; + // Counter for consecutive Submit() calls that skipped the kernel // entry (no prepared SQEs and IORING_SQ_TASKRUN not set). When the // ring is configured with IORING_SETUP_DEFER_TASKRUN, the kernel @@ -1046,6 +1261,35 @@ class IouringMgr : public AsyncIoManager // there's nothing to do. static constexpr uint32_t kForceSubmitEveryNoOps = 10; uint32_t consecutive_skipped_submits_{0}; + // SQEs handed out minus CQEs reaped (single shard thread). + uint32_t inflight_ios_{0}; + + // Loop-behavior stats, maintained by the owning shard thread and + // flushed as one VLOG(1) line every 5s: iteration rate, CQE rate, + // and the reap-batch-size histogram (how many CQEs each non-empty + // PollComplete found). Used to observe completion-batching regimes. + // Stamped once at PollComplete entry and shared by that round's CQEs and + // loop statistics. + uint64_t loop_now_us_{0}; + uint64_t stats_next_flush_us_{0}; + uint64_t stats_iters_{0}; + uint64_t stats_polls_nonzero_{0}; + uint64_t stats_cqes_{0}; + uint64_t stats_batch_hist_[9]{}; // index 1..8 = batch size, 0 = 9+ + // Per-stage read-path timing (ELOQ_IO_STATS=1): [0]=budget-gate wait, + // [1]=SQE->CQE (pre-dispatch + device + reap), [2]=CQE->task resume, + // [3]=enqueue->dequeue (queue wait), [4]=task-start->gate + // (index/root walk), [5]=dequeue->task-start (start lag). + bool io_stats_enabled_{false}; + uint64_t stage_sum_us_[6]{}; + uint64_t stage_max_us_[6]{}; + // Loop round-length tracking (gap between successive PollComplete + // calls on this shard) while stats are enabled. + uint64_t round_prev_us_{0}; + uint64_t round_sum_us_{0}; + uint64_t round_max_us_{0}; + uint64_t round_cnt_{0}; + uint64_t stage_cnt_{0}; // Active branch for this shard. std::string active_branch_{MainBranchName}; @@ -1135,6 +1379,71 @@ class IouringMgr : public AsyncIoManager return tail_scratch_acquire_count_; } + IoQosStats GetIoQosStats() const override + { + IoQosStats stats; + stats.fdatasync_count_ = + fdatasync_count_.load(std::memory_order_relaxed); + stats.fdatasync_us_ = fdatasync_us_.load(std::memory_order_relaxed); + stats.rate_ = rate_budget_.Stats(); + stats.bg_rate_ = rate_budget_.BgStats(); + stats.io_window_inflight_ = + io_window_inflight_.load(std::memory_order_relaxed); + stats.io_window_hwm_ = io_window_hwm_.load(std::memory_order_relaxed); + stats.io_window_blocked_ = + io_window_blocked_.load(std::memory_order_relaxed); + return stats; + } + + /** + * @brief Single class-blind cap on in-flight device commands + * (KvOptions::max_inflight_io; 0 = off). Companion to the rate budget: + * rate governs allocation per second with class policy; this bounds + * the instantaneous outstanding window toward the device, smoothing + * the rate bucket's burst release. Cost is device commands: 1 per page + * IO, ceil(len / kDeviceCmdBytes) per merged write, 1 per batch fsync + * (FdatasyncFiles — a flush occupies a queue slot like any command). + * Oversized-request escape: a cost above the cap admits alone once the + * window drains. Completion-driven wake in PollComplete. + */ + void AcquireIoWindow(uint32_t cost); + void ReleaseIoWindow(uint32_t cost); + /** + * @brief Kernel splits large submissions into device commands of at + * most this size (queue-depth currency; distinct from + * rate_limit_io_unit, the hypervisor-accounting currency). + */ + static constexpr uint32_t kDeviceCmdBytes = 256 * 1024; + /** + * @brief In-flight-window cost of a submission of @p bytes, in device + * commands. Acquire and release must use the same formula. + */ + static uint32_t DeviceCmdCost(size_t bytes) + { + return static_cast((bytes + kDeviceCmdBytes - 1) / + kDeviceCmdBytes); + } + + /** + * @brief Rate-budget ops cost of a write of @p bytes, in + * rate_limit_io_unit-sized device-accounting operations + * (ceil(bytes / unit)). The single source of the write ops charge — + * both WritePage (one data page) and SubmitMergedWrite use it, so a + * page's cost and a merged write's cost are consistent by construction. + * The unit is the configured physical write quantum (default = data + * page size); EloqStore::ValidateOptions rejects units below 4KB — + * covering programmatic KvOptions, not just the INI path — so the + * divisor is never zero and no clamp is needed here. + */ + static uint32_t WriteRateOpsFor(size_t bytes, uint32_t unit) + { + return static_cast((bytes + unit - 1) / unit); + } + uint32_t WriteRateOps(size_t bytes) const + { + return WriteRateOpsFor(bytes, options_->rate_limit_io_unit); + } + /** * @brief Pure-function form of BufIndexForAddress: searches @p chunks for * the one containing @p ptr and returns base_index + chunk_index, or diff --git a/include/eloq_store.h b/include/eloq_store.h index d09bce2b..221351c9 100644 --- a/include/eloq_store.h +++ b/include/eloq_store.h @@ -141,6 +141,10 @@ class KvRequest void SetTableId(TableIdent tbl_id); const TableIdent &TableId() const; uint64_t UserData() const; + // Stage-timing instrumentation only (ELOQ_IO_STATS=1): microsecond + // timestamp when the current attempt was enqueued to its shard. + uint64_t dbg_enqueue_us_{0}; + uint64_t dbg_dequeue_us_{0}; /** * @brief Test if this request is done. @@ -979,6 +983,15 @@ class EloqStore */ size_t TailScratchAcquireCount(size_t shard_id) const; + /** + * @brief Per-shard IO QoS statistics (in-flight page-IO budgets, + * fdatasync accounting; see docs/design/io_qos.md). Backing counters use + * single-writer relaxed atomics; the returned plain value is a race-free + * but non-coherent live snapshot and is exact once the shard is quiesced. + * Returns zeros for invalid shard IDs or managers without budgets. + */ + IoQosStats GetIoQosStats(size_t shard_id) const; + bool ExecAsyn(KvRequest *req); void ExecSync(KvRequest *req); diff --git a/include/eloqstore_metrics.h b/include/eloqstore_metrics.h index b19de54a..c1c99dda 100644 --- a/include/eloqstore_metrics.h +++ b/include/eloqstore_metrics.h @@ -47,10 +47,15 @@ inline const Name NAME_ELOQSTORE_OPEN_FILE_LIMIT{"eloqstore_open_file_limit"}; inline const Name NAME_ELOQSTORE_LOCAL_SPACE_USED{"eloqstore_local_space_used"}; inline const Name NAME_ELOQSTORE_LOCAL_SPACE_LIMIT{ "eloqstore_local_space_limit"}; +inline const Name NAME_ELOQSTORE_INFLIGHT_READ_PAGES{ + "eloqstore_inflight_read_pages"}; +inline const Name NAME_ELOQSTORE_INFLIGHT_BG_READ_PAGES{ + "eloqstore_inflight_bg_read_pages"}; +inline const Name NAME_ELOQSTORE_INFLIGHT_WRITE_PAGES{ + "eloqstore_inflight_write_pages"}; -// Collection interval for Phase 9-11 gauge metrics (index buffer pool, page -// pool, open file, local space) These metrics are collected every N -// WorkOneRound() calls to reduce overhead +// Periodic scheduler gauges are collected every N active shard rounds to +// reduce overhead. inline constexpr size_t ELOQSTORE_GAUGE_COLLECTION_INTERVAL = 1000; } // namespace metrics #endif // ELOQSTORE_WITH_TXSERVICE diff --git a/include/eloqstore_module.h b/include/eloqstore_module.h index dad8bef2..ad7fa095 100644 --- a/include/eloqstore_module.h +++ b/include/eloqstore_module.h @@ -19,6 +19,11 @@ class EloqStoreModule : public eloq::EloqModule } ~EloqStoreModule() = default; + eloq::ModuleType Type() const override + { + return eloq::ModuleType::kEloqStore; + } + void ExtThdStart(int thd_id) override; void ExtThdEnd(int thd_id) override; void Process(int thd_id) override; diff --git a/include/fail_point.h b/include/fail_point.h index 90008e92..ebd5e7fe 100644 --- a/include/fail_point.h +++ b/include/fail_point.h @@ -1,18 +1,25 @@ #pragma once +#include +#include #include // Test-only error injection. Unlike KillPoint (kill_point.h), which SIGTERMs -// the process to exercise crash-recovery paths, FailPoint makes a code path -// return an error so tests can drive in-process error/abort handling (e.g. -// verifying WriteTask::Abort rolls back the BranchFileMapping high-water -// marks). +// the process to exercise crash-recovery paths, FailPoint perturbs a code path +// in-process (for example, returning an error or forcing a scheduler yield). // -// Usage: arm a named point from the test, then a TEST_FAIL_POINT_RETURN(name, -// err) embedded in engine code returns `err` exactly once before -// auto-disarming. Compiled out entirely in release (NDEBUG) builds, like the -// kill-point macros. +// Usage: ArmOnce auto-disarms after a matching TEST_FAIL_POINT_RETURN or +// TEST_FAIL_POINT_ACTION; ArmPersistent fires every match until Disarm. +// Compiled out entirely in release (NDEBUG) builds, like the kill-point macros. #ifndef NDEBUG +#define TEST_FAIL_POINT_ACTION(name, action) \ + do \ + { \ + if (::eloqstore::FailPoint::GetInstance().ShouldFail(name)) \ + { \ + action; \ + } \ + } while (0) #define TEST_FAIL_POINT_RETURN(name, err) \ do \ { \ @@ -22,6 +29,10 @@ } \ } while (0) #else +#define TEST_FAIL_POINT_ACTION(name, action) \ + do \ + { \ + } while (0) #define TEST_FAIL_POINT_RETURN(name, err) \ do \ { \ @@ -45,25 +56,80 @@ class FailPoint // @p name must be a string literal (stored by pointer, not copied). void ArmOnce(const char *name) { - armed_ = name; + Arm(name, false, false); + } + + // Arm until Disarm. Used by scheduler regressions that must perturb every + // matching wake throughout a bounded observation window. + void ArmPersistent(const char *name) + { + Arm(name, true, false); + } + + // Arm a persistent point and hold a cooperatively-yielding action at its + // first wake until the test observes the exact scheduler state. The action + // calls MarkPauseReached() only after yielding once, then polls + // PauseRequested() between further yields so it never blocks the shard. + void ArmPersistentPaused(const char *name) + { + Arm(name, true, true); + } + + void ReleasePause() + { + paused_.store(false, std::memory_order_release); + } + + bool PauseRequested() const + { + return paused_.load(std::memory_order_acquire); + } + + void MarkPauseReached() + { + pause_reached_.store(true, std::memory_order_release); + } + + bool PauseReached() const + { + return pause_reached_.load(std::memory_order_acquire); } void Disarm() { - armed_ = nullptr; + paused_.store(false, std::memory_order_release); + armed_.store(nullptr, std::memory_order_release); } bool ShouldFail(const char *name) { - if (armed_ == nullptr || std::strcmp(armed_, name) != 0) + const char *armed = armed_.load(std::memory_order_acquire); + if (armed == nullptr || std::strcmp(armed, name) != 0) { return false; } - armed_ = nullptr; // fire once - return true; + if (persistent_.load(std::memory_order_relaxed)) + { + return true; + } + return armed_.compare_exchange_strong(armed, + nullptr, + std::memory_order_acq_rel, + std::memory_order_acquire); } private: - const char *armed_{nullptr}; + void Arm(const char *name, bool persistent, bool paused) + { + pause_reached_.store(false, std::memory_order_relaxed); + paused_.store(paused, std::memory_order_relaxed); + persistent_.store(persistent, std::memory_order_relaxed); + armed_.store(name, std::memory_order_release); + } + + std::atomic armed_{nullptr}; + std::atomic persistent_{false}; + std::atomic paused_{false}; + std::atomic pause_reached_{false}; }; } // namespace eloqstore diff --git a/include/kv_options.h b/include/kv_options.h index e71d2863..cbb7d93d 100644 --- a/include/kv_options.h +++ b/include/kv_options.h @@ -73,12 +73,112 @@ struct KvOptions */ uint32_t io_queue_size = 4096; /** - * @brief Max amount of inflight write IO per shard. - * Only take effect in non-append write mode. - */ - uint32_t max_inflight_write = 32 << 10; - /** - * @brief The maximum number of pages per batch for the write task. + * @brief Write request-pool sizing (WriteReqPool / MergedWriteReqPool + * elements per shard). Cannot be zero. Historically this also acted as + * an in-flight page-write QoS cap (docs/design/io_qos.md M1); that + * role is retired — device admission control is the rate budget + * (disk_rate_limit_iops, M4) plus the optional class-blind window + * (max_inflight_io) — and the count-based cap could not bind below + * one merged write buffer anyway (256 pages at the 1MB default). + */ + uint32_t max_inflight_write = 32768; + /** + * @brief DEPRECATED — no effect. Formerly the per-shard in-flight + * page-read cap (docs/design/io_qos.md M1). Superseded by the device + * rate budget (disk_rate_limit_iops, M4), which controls the correct + * dimension on rate-metered disks, and by max_inflight_io for + * instantaneous depth bounding. Parsed with a warning for + * compatibility; scheduled for removal one release after deprecation. + */ + uint32_t max_inflight_read = 0; + /** + * @brief DEPRECATED — no effect. Formerly the background share of + * max_inflight_read (M2). Class policy lives in the rate budget's + * rate_bg_ratio, which covers background reads and all writes. + * Parsed with a warning for compatibility; scheduled for removal one + * release after deprecation. + */ + uint32_t bg_read_ratio = 25; + /** + * @brief Per-disk (per store_path) device rate limit in operations per + * second (docs/design/io_qos.md M4). 0 disables the ops bucket. Each + * shard's budget is disk_rate_limit_iops * store_path.size() / + * num_threads (multiple store paths are assumed to be identical + * devices). Set to ~90-95% of the disk's enforced ceiling (cloud disks + * are provisioned rate limits; measure with deep-queue fio or read the + * provider's documented figure) so that IO waits in the shard's + * admission queue — FIFO, foreground-first — instead of in the + * hypervisor's limiter, which holds overflow IOs in quantized multi-ms + * delays. The default is a reasonable starting point for current + * cloud-local NVMe (the measured Azure v2 direct disk ceiling); + * devices faster than ~290K IOPS are under-used until it is raised or + * disabled, and slower disks should be measured and set accordingly. + */ + uint64_t disk_rate_limit_iops = 275'000; + /** + * @brief Per-disk (per store_path) device rate limit in MB/s + * (docs/design/io_qos.md M4). 0 disables the bytes bucket. Divided + * across shards like disk_rate_limit_iops. + */ + uint64_t disk_rate_limit_mbps = 0; + /** + * @brief Rate-bucket capacity, in milliseconds of refill (M4). Bounds + * how much unspent credit can bank while a shard is idle and thus the + * largest instantaneous burst admitted after a gap; equivalently, the + * worst-case transient queueing an idle-to-busy edge adds. The window + * only reshapes the latency distribution (the mean is fixed by + * throughput): smaller = flatter (higher median, lower tail), larger = + * burstier (lower median, fatter tail). Measured 2026-07-22 at deep + * queue: 1/2/4 ms cost no throughput (within 0.5%); storm p99.9 was + * 2463/2619/2808 us and p50 382/243/133 us. Default 2 ms balances the + * two; use 1 ms when tails matter most. + */ + uint32_t rate_limit_burst_ms = 2; + /** + * @brief Ops-cost quantum for writes against the ops bucket (M4): a + * write of `len` bytes is charged ceil(len / rate_limit_io_unit) + * operations (WriteRateOps), so a data-page write and a merged write + * are metered on one consistent scale. Defaults to the physical write + * quantum, one data page (4KB): a 4KB write costs 1 op, a 1MB merged + * write costs 256. Minimum 4KB (= the default): ValidateOptions + * rejects smaller values — the divisor must never be zero, and a + * sub-page quantum would overcharge writes — and LoadFromIni keeps + * the default on a malformed or out-of-uint32-range entry. The tested + * Azure NVMe read-tail target was met at the 4KB default; coarsen + * (recalibrating with the fio boundary method in docs/design/io_qos.md) + * only if a device's write-accounting unit is measured larger. + */ + uint32_t rate_limit_io_unit = 4 * KB; + /** + * @brief Background share of the device rate budget, percent (M4). + * The rate budget is partitioned: background IO (background-task + * reads and all write-path IO) refills at this share, foreground + * reads at the remainder; covers writes as well, not only reads + * (unlike the retired count-era bg_read_ratio). Clamped to [1, 99]. + */ + uint32_t rate_bg_ratio = 25; + /** + * @brief Single class-blind cap on in-flight device commands per shard + * (M4 companion). 0 = off. Smooths the rate bucket's burst release + * toward the device: the rate budget governs allocation per second + * (with class policy), this bounds the instantaneous outstanding + * window. Deliberately does not distinguish reads/writes or + * foreground/background — by admission time the rate budget has + * already applied policy, and the device queue this replaces is + * class-blind anyway. Charged 1 per page read, ceil(len / 256KB) per + * merged write (the kernel's device-command split). Size a little + * above the throttled rate's bandwidth-delay product per shard + * (rate_per_shard x t_read, ~2x headroom). + */ + uint32_t max_inflight_io = 0; + /** + * @brief DEPRECATED — no effect. Formerly the per-write-task in-flight + * page cap (the task drained to zero via WaitWrite once it had this + * many writes outstanding). Superseded by the shard-wide write budget + * `max_inflight_write` (docs/design/io_qos.md, plan commit 4), which + * bounds in-flight write pages across all tasks without the + * drain-to-zero sawtooth. Parsed for compatibility and otherwise ignored; + * scheduled for removal one release after deprecation. */ uint32_t max_write_batch_pages = 32; /** diff --git a/include/storage/shard.h b/include/storage/shard.h index fb374009..45ab31a9 100644 --- a/include/storage/shard.h +++ b/include/storage/shard.h @@ -78,6 +78,13 @@ class Shard return DurationMicroseconds(cur_resume_start_us_); } + // Cheap TSC-based clock (rdtsc / calibrated cycles-per-us; ARM virtual + // counter on aarch64). Public so shard-thread code outside Shard (e.g. + // RateBudget blocked-time accounting) can time intervals without a + // clock_gettime call. + static uint64_t ReadTimeMicroseconds(); + uint64_t DurationMicroseconds(uint64_t start_us); + std::atomic io_mgr_and_page_pool_inited_{false}; #ifdef ELOQ_MODULE_ENABLED @@ -97,6 +104,9 @@ class Shard private: void WorkLoop(); void InitIoMgrAndPagePool(); +#ifdef ELOQSTORE_WITH_TXSERVICE + void CollectPeriodicGauges(metrics::Meter *meter); +#endif bool ExecuteReadyTasks(); void OnTaskFinished(KvTask *task); void RetryOomRequest(KvRequest *req); @@ -144,34 +154,53 @@ class Shard // module worker (WorkOneRound), the sole consumer of requests_. void DrainPendingRequests(); - uint64_t ReadTimeMicroseconds(); - - uint64_t DurationMicroseconds(uint64_t start_us); - + /** + * @brief Create a task's coroutine SUSPENDED and admit it through + * ready_tasks_ instead of executing its first segment inline. + * + * The coroutine body yields straight back to the creator before running + * the request lambda, so callcc only pays the creation prologue here; + * the scheduler's ordinary resume in ExecuteReadyTasks runs the first + * segment. This makes ready_tasks_ the single scheduling point for new + * and in-flight work: previously each dequeued request executed its + * first segment inline during intake (up to 128 per loop iteration), + * ahead of tasks already resumed by IO completions or budget grants, + * which gradually starved mid-flight tasks under high load and inflated + * tail latency. + */ template void StartTask(KvTask *task, KvRequest *req, F lbd) { task->req_ = req; task->result_err_ = KvError::NoError; task->status_ = TaskStatus::Ongoing; +#ifdef ELOQSTORE_WITH_TXSERVICE + // Captured at creation so the measured request duration includes the + // ready-queue wait (before the suspended-creation change, creation + // and first execution were the same instant). + metrics::TimePoint request_start{}; + if (this->store_->EnableMetrics()) + { + request_start = metrics::Clock::now(); + } +#endif running_ = task; - // Mark the resume start so a cooperative background loop measures this - // first segment from now, not from the previously resumed task's - // timestamp (see CurResumeElapsedUs / MaybeYield). - cur_resume_start_us_ = ReadTimeMicroseconds(); task->coro_ = boost::context::callcc( std::allocator_arg, stack_allocator_, - [lbd, this](continuation &&sink) - { + [lbd, + this #ifdef ELOQSTORE_WITH_TXSERVICE - metrics::TimePoint request_start; - if (this->store_->EnableMetrics()) - { - request_start = metrics::Clock::now(); - } + , + request_start #endif + ](continuation &&sink) + { shard->main_ = std::move(sink); + // Created suspended: hand control straight back to StartTask + // without running the request body; the scheduler's first + // resume() continues from here. + shard->main_ = shard->main_.resume(); KvError err = lbd(); KvTask *task = ThdTask(); task->result_err_ = err; @@ -189,10 +218,10 @@ class Shard return std::move(shard->main_); }); running_ = nullptr; - if (task->status_ == TaskStatus::Finished) - { - OnTaskFinished(task); - } + // The prologue cannot finish the task; it is admitted like any + // resumed task, in arrival order relative to them. + assert(task->status_ == TaskStatus::Ongoing); + ready_tasks_.Enqueue(task); } moodycamel::BlockingConcurrentQueue requests_; @@ -282,9 +311,8 @@ class Shard #endif #ifdef ELOQSTORE_WITH_TXSERVICE - size_t work_one_round_count_{ - 0}; // Counter for frequency-controlled metric collection (not atomic - // since each Shard runs in single-threaded context) + // Not atomic: both scheduler modes execute one shard serially. + size_t gauge_collection_round_count_{0}; #endif // TSC frequency in cycles per microsecond (measured at initialization) diff --git a/include/tasks/task.h b/include/tasks/task.h index e80f7c7c..ff4a9eb9 100644 --- a/include/tasks/task.h +++ b/include/tasks/task.h @@ -194,6 +194,26 @@ class KvTask { return Type() < TaskType::BatchWrite; } + /** + * @brief Background tasks' page reads are charged against the background + * sub-budget of the read IO budget (docs/design/io_qos.md M2), so they + * cannot crowd out foreground point reads at the device. Deliberately + * not derived from ReadOnly(): EvictFile and Prewarm are read-only but + * background. + */ + bool IsBackground() const + { + switch (Type()) + { + case TaskType::BatchWrite: + case TaskType::BackgroundWrite: + case TaskType::EvictFile: + case TaskType::Prewarm: + return true; + default: + return false; + } + } void Yield(); void YieldToLowPQ(); /** @@ -211,6 +231,17 @@ class KvTask int io_res_{0}; uint32_t io_flags_{0}; KvError result_err_{KvError::NoError}; + // Rate-budget waiter handshake (RateBudget::Acquire / RefillAndWake): + // the acquisition cost is recorded here at enqueue so the waker can + // peek the FIFO head and charge on its behalf before waking it. + uint32_t rate_wait_ops_{0}; + uint64_t rate_wait_bytes_{0}; + // Loop-time (us) when this task's latest page-read CQE was reaped; + // stage-timing instrumentation only (ELOQ_IO_STATS=1). + uint64_t op_cqe_us_{0}; + // Stage-timing (ELOQ_IO_STATS=1): when this task began executing its + // read; consumed (zeroed) by the first instrumented page read. + uint64_t op_start_us_{0}; TaskStatus status_{TaskStatus::Idle}; KvRequest *req_{nullptr}; @@ -224,9 +255,23 @@ class WaitingZone WaitingZone() = default; void Wait(KvTask *task); void WakeOne(); - void WakeN(size_t n); + /** + * @brief Wake up to n waiters in FIFO order. + * @return The number actually woken (< n when the zone drains), so the + * caller can forward unused wake credits to another zone. + */ + size_t WakeN(size_t n); void WakeAll(); bool Empty() const; + /** + * @brief The next waiter in FIFO order without waking it, or nullptr. + * Lets a waker peek the head's recorded cost and debit on its behalf + * before waking (see RateBudget::RefillAndWake). + */ + KvTask *Head() const + { + return head_; + } private: void PushBack(KvTask *task); diff --git a/include/types.h b/include/types.h index 01372c7b..7e28f451 100644 --- a/include/types.h +++ b/include/types.h @@ -27,6 +27,38 @@ enum class StoreMode Cloud }; +/** + * @brief Plain snapshot of one shard's IO QoS statistics. + * + * The backing counters are single-writer relaxed atomics. Sampling them is + * race-free but not coherent across fields while the shard is live; values + * are exact once the shard is quiesced. See docs/design/io_qos.md. + */ +struct IoQosStats +{ + // Device rate budget (M4). rate_ is the foreground class; bg_rate_ is + // background (background-task reads and all write-path IO), paced by + // its partitioned share. + struct Rate + { + uint64_t blocked_count_{0}; // acquisitions that had to wait + uint64_t blocked_us_{0}; // cumulative wait time + uint64_t admitted_ops_{0}; // cumulative device-op cost admitted + uint64_t admitted_bytes_{0}; // cumulative bytes admitted + // Of admitted_ops_, the ops granted from the OTHER class's idle + // surplus (borrow-when-idle; see RateBudget). + uint64_t borrowed_ops_{0}; + }; + Rate rate_; + Rate bg_rate_; + // Single class-blind in-flight device-command window (max_inflight_io). + uint32_t io_window_inflight_{0}; + uint32_t io_window_hwm_{0}; + uint64_t io_window_blocked_{0}; + uint64_t fdatasync_count_{0}; // write-path fdatasync ops (FdatasyncFiles) + uint64_t fdatasync_us_{0}; // cumulative batch wall time +}; + using PageId = uint32_t; constexpr PageId MaxPageId = UINT32_MAX; diff --git a/include/utils.h b/include/utils.h index 7a22a33e..d4449992 100644 --- a/include/utils.h +++ b/include/utils.h @@ -215,6 +215,17 @@ inline std::string BuildStorePathListWithWeights( return oss.str(); } +// Runtime gate for the IO stage-timing instrumentation (ELOQ_IO_STATS=1). +inline bool IoStatsEnabled() +{ + static const bool enabled = [] + { + const char *e = getenv("ELOQ_IO_STATS"); + return e != nullptr && e[0] == '1'; + }(); + return enabled; +} + inline std::vector ComputeStorePathLut( const std::vector &weights, size_t max_entries = kDefaultStorePathLutEntries) diff --git a/python/pyproject.toml b/python/pyproject.toml old mode 100644 new mode 100755 diff --git a/rust/eloqstore-sys/Cargo.toml b/rust/eloqstore-sys/Cargo.toml old mode 100644 new mode 100755 diff --git a/rust/eloqstore/Cargo.toml b/rust/eloqstore/Cargo.toml old mode 100644 new mode 100755 diff --git a/scripts/io_calibration_sweep.sh b/scripts/io_calibration_sweep.sh new file mode 100755 index 00000000..5b90fb6b --- /dev/null +++ b/scripts/io_calibration_sweep.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# +# IO QoS device calibration sweep (docs/design/io_qos.md, evaluation step 1). +# +# Runs a fixed 4KB random-read fio job while stepping a rate-limited +# sequential-write job through increasing MB/s targets, and emits a CSV of +# write_target_MBps, write_actual_MBps, read_iops, read_p50_us, +# read_p99_us, read_p999_us +# The knee of the p99-vs-write-rate curve is the per-device background write +# budget; divide by num_threads for the per-shard bg budget / M3 rate limit. +# +# IMPORTANT — read before trusting the numbers: +# * PRECONDITION the drive: a fresh/trimmed SSD serves reads from empty FTL +# mappings and absorbs writes into a pristine SLC cache, both of which +# flatter the results. Fill the test file region at least once (the script +# lays out the file with a full sequential write pass unless it already +# exists) and ideally run `--precondition` (two full overwrite passes) +# on a drive that has seen real use. +# * SLC-CACHE EXHAUSTION: consumer drives fold SLC->TLC once the cache +# fills; read tails get much worse and unstable after that point. Size +# per-step runtime (--step-secs, default 60) and the write rates so the +# total bytes written per step exceed the SLC cache if you want +# steady-state numbers; watch for a step whose actual write MB/s sags +# below the target — that is the fold-over signature. +# * The script always runs against files below --dir, so the result includes +# filesystem effects. Use a dedicated directory on the target device. +# +# Requires: fio, python3. +# +# Usage: +# scripts/io_calibration_sweep.sh --dir /mnt/nvme/fio-test \ +# [--size 32G] [--step-secs 60] [--read-qd 32] \ +# [--rates "0 50 100 200 400 800 1600"] [--precondition] + +set -euo pipefail + +DIR="" +SIZE="32G" +STEP_SECS=60 +READ_QD=32 +RATES="0 50 100 200 400 800 1600" +PRECONDITION=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --dir) DIR="$2"; shift 2 ;; + --size) SIZE="$2"; shift 2 ;; + --step-secs) STEP_SECS="$2"; shift 2 ;; + --read-qd) READ_QD="$2"; shift 2 ;; + --rates) RATES="$2"; shift 2 ;; + --precondition) PRECONDITION=1; shift ;; + *) echo "unknown arg: $1" >&2; exit 1 ;; + esac +done + +[[ -n "$DIR" ]] || { echo "usage: $0 --dir [...]" >&2; exit 1; } +command -v fio >/dev/null || { echo "fio not installed" >&2; exit 1; } +command -v python3 >/dev/null || { echo "python3 not installed" >&2; exit 1; } + +mkdir -p "$DIR" +READ_FILE="$DIR/calib_read.bin" +WRITE_FILE="$DIR/calib_write.bin" + +layout() { + local file="$1" passes="$2" + for ((i = 0; i < passes; i++)); do + echo "# layout pass $((i + 1))/$passes of $file" >&2 + fio --name=layout --filename="$file" --size="$SIZE" --rw=write \ + --bs=1M --iodepth=8 --direct=1 --ioengine=libaio \ + --output-format=terse >/dev/null + done +} + +# Lay out both files so reads never hit holes; precondition = extra passes. +PASSES=$((PRECONDITION ? 2 : 1)) +[[ -f "$READ_FILE" && $PRECONDITION -eq 0 ]] || layout "$READ_FILE" "$PASSES" +[[ -f "$WRITE_FILE" && $PRECONDITION -eq 0 ]] || layout "$WRITE_FILE" "$PASSES" + +echo "write_target_MBps,write_actual_MBps,read_iops,read_p50_us,read_p99_us,read_p999_us" + +for rate in $RATES; do + OUT=$(mktemp) + if [[ "$rate" == "0" ]]; then + # Baseline: reads only. + fio --output-format=json --output="$OUT" \ + --name=randread --filename="$READ_FILE" --size="$SIZE" \ + --rw=randread --bs=4k --iodepth="$READ_QD" --direct=1 \ + --ioengine=libaio --time_based --runtime="$STEP_SECS" \ + >/dev/null + else + fio --output-format=json --output="$OUT" \ + --name=randread --filename="$READ_FILE" --size="$SIZE" \ + --rw=randread --bs=4k --iodepth="$READ_QD" --direct=1 \ + --ioengine=libaio --time_based --runtime="$STEP_SECS" \ + --name=seqwrite --filename="$WRITE_FILE" --size="$SIZE" \ + --rw=write --bs=1M --iodepth=4 --direct=1 \ + --ioengine=libaio --time_based --runtime="$STEP_SECS" \ + --rate=,"${rate}m" \ + >/dev/null + fi + python3 - "$OUT" "$rate" <<'EOF' +import json +import sys + +with open(sys.argv[1]) as f: + data = json.load(f) +rate = sys.argv[2] +read_iops = 0.0 +read_p = {"50.000000": 0, "99.000000": 0, "99.900000": 0} +write_mbps = 0.0 +for job in data["jobs"]: + if job["jobname"] == "randread": + r = job["read"] + read_iops = r["iops"] + # clat_ns percentiles -> us + pct = r.get("clat_ns", {}).get("percentile", {}) + for k in read_p: + read_p[k] = pct.get(k, 0) / 1000.0 + elif job["jobname"] == "seqwrite": + write_mbps = job["write"]["bw_bytes"] / (1 << 20) +print( + f"{rate},{write_mbps:.0f},{read_iops:.0f}," + f"{read_p['50.000000']:.0f},{read_p['99.000000']:.0f}," + f"{read_p['99.900000']:.0f}" +) +EOF + rm -f "$OUT" +done diff --git a/src/async_io_manager.cpp b/src/async_io_manager.cpp index 8e676ab9..64657c4c 100644 --- a/src/async_io_manager.cpp +++ b/src/async_io_manager.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -166,16 +167,324 @@ bool AsyncIoManager::IsIdle() return true; } +// Split a per-shard total rate into (foreground, background) shares. A +// rate of 0 means "dimension disabled" and is preserved as such. For any +// enabled dimension both shares are kept nonzero so integer truncation at +// low totals cannot silently disable a class — a zero background rate +// would leave background writes unbudgeted, a zero foreground rate would +// drop read protection. The sum equals the total exactly for total >= 2; +// a total of 1 cannot fund two integer shares, so both are set to 1 (a +// one-unit overshoot at a degenerate rate) and logged. +static void SplitRate(uint64_t total, + uint32_t bg_ratio_pct, + const char *dim, + uint64_t &fg, + uint64_t &bg) +{ + if (total == 0) + { + fg = 0; + bg = 0; + return; + } + if (total < 2) + { + LOG(WARNING) << "per-shard " << dim << " rate " << total + << " too low to partition into foreground/background; " + "using 1/1 to keep both classes nonzero"; + fg = 1; + bg = 1; + return; + } + bg = std::clamp(total * bg_ratio_pct / 100, 1, total - 1); + fg = total - bg; +} + +void RateBudget::SetRates(uint64_t ops_per_sec, + uint64_t bytes_per_sec, + uint32_t burst_ms, + uint32_t bg_ratio_pct) +{ + // Partition the total rate between the classes (see class comment): + // background gets ratio percent, foreground the rest, both kept nonzero + // for any enabled dimension (see SplitRate). + const uint32_t ratio = std::clamp(bg_ratio_pct, 1, 99); + SplitRate(ops_per_sec, ratio, "ops", fg_ops_rate_, bg_ops_rate_); + SplitRate(bytes_per_sec, ratio, "bytes", fg_bytes_rate_, bg_bytes_rate_); + burst_us_ = uint64_t{std::max(burst_ms, 1)} * 1000; +} + +bool RateBudget::Positive(bool background) const +{ + if (background) + { + return (bg_ops_rate_ == 0 || bg_ops_bal_ > 0) && + (bg_bytes_rate_ == 0 || bg_bytes_bal_ > 0); + } + return (fg_ops_rate_ == 0 || fg_ops_bal_ > 0) && + (fg_bytes_rate_ == 0 || fg_bytes_bal_ > 0); +} + +bool RateBudget::CanAdmit(bool background) const +{ + if (Positive(background)) + { + return true; + } + if (background) + { + // Background never borrows. Symmetric borrowing was tried and + // reverted (2026-07-22): a closed-loop foreground's waiting zone + // empties for microseconds between completion and resubmission, + // and in those windows storm-driven background (whose demand is + // effectively unbounded) skimmed the foreground refill wholesale — + // measured as bg_rate_borrowed ~2M ops/shard per storm, foreground + // QPS 183K -> 116K, p99.9 720 µs -> 5.6 ms. Foreground's share is + // a guarantee AGAINST background; it must hold no matter how idle + // foreground momentarily looks. + return false; + } + // Foreground may borrow background's surplus while background has no + // queued demand. Requiring a positive lender balance means surplus is + // lent, never debt — background restarts at worst one IO-cost below + // zero when its demand returns. Lending in this direction can only + // reduce device contention for the class the tail contract protects. + return bg_waiting_.Empty() && Positive(true); +} + +void IouringMgr::AcquireIoWindow(uint32_t cost) +{ + if (io_window_cap_ == 0) + { + return; + } + // Class-blind FIFO with the oversized-request escape (cost above the + // cap admits alone once the window drains). + // Queue behind existing waiters so arrival order is preserved. + auto inflight = [this] + { return io_window_inflight_.load(std::memory_order_relaxed); }; + if (!io_window_waiting_.Empty() || + (inflight() + cost > io_window_cap_ && inflight() != 0)) + { + io_window_blocked_.fetch_add(1, std::memory_order_relaxed); + do + { + io_window_waiting_.Wait(ThdTask()); + } while (inflight() + cost > io_window_cap_ && inflight() != 0); + } + const uint32_t now_inflight = inflight() + cost; + io_window_inflight_.store(now_inflight, std::memory_order_relaxed); + if (now_inflight > io_window_hwm_.load(std::memory_order_relaxed)) + { + io_window_hwm_.store(now_inflight, std::memory_order_relaxed); + } +} + +void IouringMgr::ReleaseIoWindow(uint32_t cost) +{ + if (io_window_cap_ == 0) + { + return; + } + const uint32_t cur = io_window_inflight_.load(std::memory_order_relaxed); + assert(cur >= cost); + io_window_inflight_.store(cur - cost, std::memory_order_relaxed); + // Each freed command can admit at most one waiter; over-waking is safe + // (woken tasks re-check and re-wait). + io_window_waiting_.WakeN(cost); +} + +void RateBudget::Charge(uint32_t ops, uint64_t bytes, bool background) +{ + // Debt semantics: subtract the full cost, letting the balance go + // negative. Callers guarantee CanAdmit(background) held at the moment + // of the charge. A balance is debited ONLY when its dimension is + // enabled (rate != 0): a disabled dimension is never refilled and + // never consulted by Positive(), so debiting it would drive it + // monotonically negative to int64 overflow on a long-lived shard. + const int64_t ops_cost = int64_t{ops} * kScale; + const int64_t bytes_cost = static_cast(bytes) * kScale; + if (background) + { + // Background admits only on its own positive balances (CanAdmit + // never lets it borrow), so a background charge always debits the + // background buckets — the foreground debit is unreachable by + // construction, not by luck. + assert(Positive(true)); + if (bg_ops_rate_ != 0) + { + bg_ops_bal_ -= ops_cost; + } + if (bg_bytes_rate_ != 0) + { + bg_bytes_bal_ -= bytes_cost; + } + bg_admitted_ops_.fetch_add(ops, std::memory_order_relaxed); + bg_admitted_bytes_.fetch_add(bytes, std::memory_order_relaxed); + return; + } + // Foreground: exhausted own balances mean this admission was granted + // from background's idle surplus (CanAdmit's borrow arm) — the debit + // lands on the lender so its refill repays the loan. + const bool borrowed = !Positive(false); + if (borrowed) + { + if (bg_ops_rate_ != 0) + { + bg_ops_bal_ -= ops_cost; + } + if (bg_bytes_rate_ != 0) + { + bg_bytes_bal_ -= bytes_cost; + } + } + else + { + if (fg_ops_rate_ != 0) + { + fg_ops_bal_ -= ops_cost; + } + if (fg_bytes_rate_ != 0) + { + fg_bytes_bal_ -= bytes_cost; + } + } + admitted_ops_.fetch_add(ops, std::memory_order_relaxed); + admitted_bytes_.fetch_add(bytes, std::memory_order_relaxed); + if (borrowed) + { + borrowed_ops_.fetch_add(ops, std::memory_order_relaxed); + } +} + +void RateBudget::Acquire(uint32_t ops, uint64_t bytes, bool background) +{ + if (!Enabled()) + { + return; + } + WaitingZone &zone = background ? bg_waiting_ : waiting_; + if (zone.Empty() && CanAdmit(background)) + { + // Fast path: no queue and an admissible balance — charge inline. + Charge(ops, bytes, background); + return; + } + // Queue behind existing waiters of the class (FIFO within class) with + // the cost recorded for the waker: RefillAndWake peeks the FIFO head, + // charges its recorded cost and only then wakes it, so wake counts + // are exact for heterogeneous costs and a woken task never re-queues. + // GrantWaiters is the ONLY waker of these zones, so by the time this + // task resumes its cost has already been charged and there is nothing + // left to do here; a second wake source must not be added without + // extending this handshake. + KvTask *task = ThdTask(); + task->rate_wait_ops_ = ops; + task->rate_wait_bytes_ = bytes; + const uint64_t start_us = shard->ReadTimeMicroseconds(); + (background ? bg_blocked_count_ : blocked_count_) + .fetch_add(1, std::memory_order_relaxed); + zone.Wait(task); + (background ? bg_blocked_us_ : blocked_us_) + .fetch_add(shard->DurationMicroseconds(start_us), + std::memory_order_relaxed); +} + +void RateBudget::RefillAndWake(uint64_t now_us) +{ + if (now_us <= last_refill_us_) + { + return; + } + // The elapsed time is clamped to the burst window, which has three + // effects. (1) An idle gap banks at most one bucket of credit: a shard + // idle for a second does not come back with a second's worth of + // tokens, only rate x burst_us_. (2) The very first call is well + // defined: last_refill_us_ is 0, so the raw elapsed would be the + // absolute clock value; clamped, the budget simply starts with one + // full bucket of allowance. (3) rate x elapsed cannot overflow int64 + // no matter how long the gap. + const uint64_t elapsed = std::min(now_us - last_refill_us_, burst_us_); + last_refill_us_ = now_us; + auto refill = [elapsed, this](int64_t &bal, uint64_t rate) + { + if (rate == 0) + { + return; + } + const int64_t cap = static_cast(rate * burst_us_); + bal = std::min(bal + static_cast(rate * elapsed), cap); + }; + refill(fg_ops_bal_, fg_ops_rate_); + refill(fg_bytes_bal_, fg_bytes_rate_); + refill(bg_ops_bal_, bg_ops_rate_); + refill(bg_bytes_bal_, bg_bytes_rate_); + // The classes have disjoint buckets, so grant each independently; the + // admission test includes the borrow path, so a foreground waiter + // whose lender just became idle-and-positive is granted too. + GrantWaiters(false); + GrantWaiters(true); +} + +void RateBudget::GrantWaiters(bool background) +{ + // Peek-and-grant: charge the FIFO head's recorded cost on its behalf, + // mark it granted, then wake it — exact wake counts with + // heterogeneous costs, no over-waking, no re-queue churn. The debt + // rule is preserved verbatim: each charge may drive the balance + // negative, and the loop stops exactly where serialized admission + // would. + WaitingZone &zone = background ? bg_waiting_ : waiting_; + while (!zone.Empty() && CanAdmit(background)) + { + KvTask *head = zone.Head(); + Charge(head->rate_wait_ops_, head->rate_wait_bytes_, background); + zone.WakeOne(); + } +} + IouringMgr::IouringMgr(const KvOptions *opts, uint32_t fd_limit) : AsyncIoManager(opts), fd_limit_(fd_limit) { memset(&ring_, 0, sizeof(ring_)); + io_stats_enabled_ = IoStatsEnabled(); lru_fd_head_.next_ = &lru_fd_tail_; lru_fd_tail_.prev_ = &lru_fd_head_; uint32_t pool_size = options_->max_inflight_write; write_req_pool_ = std::make_unique(pool_size); merged_write_req_pool_ = std::make_unique(pool_size); + + // Device admission control (docs/design/io_qos.md M4): the rate budget + // (below) plus the optional class-blind in-flight command window. The + // M1/M2 count budgets are retired; max_inflight_read / bg_read_ratio + // are deprecated no-ops and max_inflight_write only sizes the request + // pools above. + io_window_cap_ = options_->max_inflight_io; + + // Device rate budget (docs/design/io_qos.md M4): the per-disk + // provisioned limits, spread across shards by simple division. Assumes + // shards spread IO uniformly across store paths (true on average via + // the store-path LUT). + if (options_->disk_rate_limit_iops != 0 || + options_->disk_rate_limit_mbps != 0) + { + const uint64_t num_disks = + std::max(1, options_->store_path.size()); + const uint64_t shards = std::max(1, options_->num_threads); + const uint64_t shard_ops = + options_->disk_rate_limit_iops * num_disks / shards; + const uint64_t shard_bytes = options_->disk_rate_limit_mbps * + num_disks * (uint64_t{1} << 20) / shards; + rate_budget_.SetRates(shard_ops, + shard_bytes, + options_->rate_limit_burst_ms, + options_->rate_bg_ratio); + LOG(INFO) << "IO rate budget: " << shard_ops << " ops/s, " + << (shard_bytes >> 20) << " MB/s per shard (" + << options_->disk_rate_limit_iops << " iops x " << num_disks + << " disks / " << shards << " shards)"; + } } IouringMgr::~IouringMgr() @@ -739,7 +1048,23 @@ std::pair IouringMgr::ReadPage(const TableIdent &tbl_id, int res; do { - io_uring_sqe *sqe = GetSQE(UserDataType::KvTask, ThdTask()); + // Read-budget admission (io_qos.md M1/M2): acquired last, right + // before SQE prep; released per CQE in PollComplete, so each + // retry iteration re-acquires. Background tasks are additionally + // bounded by the BG sub-budget. + const uint64_t t_gate = + io_stats_enabled_ ? shard->ReadTimeMicroseconds() : 0; + // Rate-budget admission (M4) before the count budget: the rate + // bucket paces device ops/bytes per second; the count budget + // stays as the burst-depth guard closest to SQE prep. Both are + // coroutine waits with independent wake sources (time vs CQE), + // so ordering cannot deadlock. + rate_budget_.Acquire( + 1, options_->data_page_size, ThdTask()->IsBackground()); + AcquireIoWindow(1); + const uint64_t t_sqe = + io_stats_enabled_ ? shard->ReadTimeMicroseconds() : 0; + io_uring_sqe *sqe = GetSQE(UserDataType::KvTaskPageRead, ThdTask()); if (fd.second) { sqe->flags |= IOSQE_FIXED_FILE; @@ -762,6 +1087,46 @@ std::pair IouringMgr::ReadPage(const TableIdent &tbl_id, sqe, fd.first, dst, options_->data_page_size, offset); } res = ThdTask()->WaitIoResult(); + if (io_stats_enabled_) + { + const uint64_t t_res = shard->ReadTimeMicroseconds(); + const uint64_t cqe = ThdTask()->op_cqe_us_; + const uint64_t t_task = ThdTask()->op_start_us_; + if (t_task != 0) + { + ThdTask()->op_start_us_ = 0; // first page read only + KvRequest *r = ThdTask()->req_; + if (r != nullptr && r->dbg_enqueue_us_ != 0 && + r->dbg_dequeue_us_ >= r->dbg_enqueue_us_ && + t_task >= r->dbg_dequeue_us_) + { + const uint64_t d3 = + r->dbg_dequeue_us_ - r->dbg_enqueue_us_; + const uint64_t d5 = t_task - r->dbg_dequeue_us_; + stage_sum_us_[3] += d3; + stage_max_us_[3] = std::max(stage_max_us_[3], d3); + stage_sum_us_[5] += d5; + stage_max_us_[5] = std::max(stage_max_us_[5], d5); + } + if (t_gate > t_task) + { + const uint64_t d4 = t_gate - t_task; + stage_sum_us_[4] += d4; + stage_max_us_[4] = std::max(stage_max_us_[4], d4); + } + } + const uint64_t d0 = t_sqe - t_gate; // gate wait + const uint64_t d1 = cqe > t_sqe ? cqe - t_sqe : 0; // sqe->cqe + const uint64_t d2 = + t_res > cqe ? t_res - cqe : 0; // cqe->resume + stage_sum_us_[0] += d0; + stage_sum_us_[1] += d1; + stage_sum_us_[2] += d2; + stage_max_us_[0] = std::max(stage_max_us_[0], d0); + stage_max_us_[1] = std::max(stage_max_us_[1], d1); + stage_max_us_[2] = std::max(stage_max_us_[2], d2); + ++stage_cnt_; + } if (res == 0) { LOG(ERROR) << "read page failed, reach end of file, file id:" @@ -838,8 +1203,15 @@ KvError IouringMgr::ReadPages(const TableIdent &tbl_id, auto send_req = [this](ReadReq *req) { + // Read-budget admission (io_qos.md M1/M2) is per page, not per batch: + // the task may block mid-batch while already-submitted pages + // complete, so a batch larger than the (sub-)budget cannot deadlock. + // Rate budget (M4) first, same per-page granularity. + rate_budget_.Acquire( + 1, options_->data_page_size, req->task_->IsBackground()); + AcquireIoWindow(1); auto [fd, registered] = req->fd_ref_.FdPair(); - io_uring_sqe *sqe = GetSQE(UserDataType::BaseReq, req); + io_uring_sqe *sqe = GetSQE(UserDataType::BaseReqPageRead, req); if (registered) { sqe->flags |= IOSQE_FIXED_FILE; @@ -983,6 +1355,15 @@ KvError IouringMgr::WritePage(const TableIdent &tbl_id, auto [fd, registered] = fd_ref.FdPair(); WriteReq *req = write_req_pool_->Alloc(std::move(fd_ref), std::move(page)); + // Device admission (io_qos.md M4): after every other blocking + // acquisition (FD, req pool), immediately before SQE prep. Rate budget + // first; write tasks classify as background. The ops cost uses the same + // WriteRateOps helper as the merged-write path so a page and a merged + // write are charged on one consistent scale. + rate_budget_.Acquire(WriteRateOps(options_->data_page_size), + options_->data_page_size, + ThdTask()->IsBackground()); + AcquireIoWindow(1); io_uring_sqe *sqe = GetSQE(UserDataType::WriteReq, req); if (registered) { @@ -1288,6 +1669,13 @@ KvError IouringMgr::SubmitMergedWrite(const TableIdent &tbl_id, static_cast(req->pages_.size() - 1); } + // Device admission (io_qos.md M4). Rate budget first: ops cost is + // ceil(bytes / rate_limit_io_unit) via WriteRateOps (the same helper + // WritePage uses), charged against the configured write quantum; the + // bytes bucket is charged the full length. Debt admission means this + // single large acquisition never deadlocks against the bucket size. + rate_budget_.Acquire(WriteRateOps(bytes), bytes, ThdTask()->IsBackground()); + AcquireIoWindow(DeviceCmdCost(bytes)); io_uring_sqe *sqe = GetSQE(UserDataType::MergedWriteReq, req); auto [fd, registered] = req->fd_ref_.FdPair(); if (registered) @@ -1940,6 +2328,14 @@ std::pair IouringMgr::ConvFileSegmentId( void IouringMgr::Submit() { + // Refill the device rate budget once per loop iteration (M4). Must run + // on every iteration — including no-op ones — because refill is the + // only wake source for rate-budget waiters. + if (rate_budget_.Enabled()) + { + rate_budget_.RefillAndWake(Shard::ReadTimeMicroseconds()); + } + const uint32_t prepared_before = prepared_sqe_; const uint32_t sq_flags = ring_.sq.kflags == nullptr ? 0 : *ring_.sq.kflags; const bool need_taskrun = (sq_flags & IORING_SQ_TASKRUN) != 0; @@ -2005,8 +2401,35 @@ void IouringMgr::Submit() } } +void IouringMgr::FlushSubmit() +{ + if (prepared_sqe_ == 0) + { + // Nothing was prepared after the round's Submit. Kernel re-entry + // (and with it the DEFER_TASKRUN forced-enter safety net) is + // Submit's job at the top of the next round, so deliberately do + // not touch consecutive_skipped_submits_ here: this call must not + // perturb that cadence. + return; + } + // A round that prepared SQEs is entering the kernel now, so the skip + // counter restarts exactly as it would in Submit. + consecutive_skipped_submits_ = 0; + int ret = io_uring_submit(&ring_); + if (__builtin_expect(ret < 0, 0)) + { + LOG(ERROR) << "iouring flush submit failed " << ret; + return; + } + prepared_sqe_ -= ret; +} + void IouringMgr::PollComplete() { + if (io_stats_enabled_) + { + loop_now_us_ = Shard::ReadTimeMicroseconds(); + } io_uring_cqe *cqe = nullptr; io_uring_peek_cqe(&ring_, &cqe); unsigned head; @@ -2019,14 +2442,38 @@ void IouringMgr::PollComplete() KvTask *task = nullptr; switch (type) { + case UserDataType::KvTaskPageRead: case UserDataType::KvTask: task = static_cast(ptr); + if (type == UserDataType::KvTaskPageRead) + { + TEST_FAIL_POINT_ACTION("KvTaskPageReadCqe", cqe->res = -EIO); + ReleaseIoWindow(1); + if (io_stats_enabled_) + { + task->op_cqe_us_ = loop_now_us_; + } + } task->io_res_ = cqe->res; task->io_flags_ = cqe->flags; break; + case UserDataType::BaseReqPageRead: + case UserDataType::BaseReqFsync: case UserDataType::BaseReq: { BaseReq *req = static_cast(ptr); + if (type == UserDataType::BaseReqPageRead) + { + TEST_FAIL_POINT_ACTION("BaseReqPageReadCqe", cqe->res = -EIO); + ReleaseIoWindow(1); + } + else if (type == UserDataType::BaseReqFsync) + { + TEST_FAIL_POINT_ACTION("BaseReqFsyncCqe", cqe->res = -EIO); + // Mirrors the per-SQE acquire in FdatasyncFiles; + // unconditional so failed CQEs release their command too. + ReleaseIoWindow(1); + } req->res_ = cqe->res; req->flags_ = cqe->flags; task = req->task_; @@ -2035,6 +2482,7 @@ void IouringMgr::PollComplete() case UserDataType::WriteReq: { WriteReq *req = static_cast(ptr); + TEST_FAIL_POINT_ACTION("WriteReqCqe", cqe->res = -EIO); KvError err; assert(cqe->res <= options_->data_page_size); if (cqe->res < 0) @@ -2052,11 +2500,13 @@ void IouringMgr::PollComplete() req->task_->WritePageCallback(std::move(req->page_), err); task = req->task_; write_req_pool_->Free(req); + ReleaseIoWindow(1); break; } case UserDataType::MergedWriteReq: { MergedWriteReq *req = static_cast(ptr); + TEST_FAIL_POINT_ACTION("MergedWriteReqCqe", cqe->res = -EIO); KvError err; if (cqe->res < 0) { @@ -2085,6 +2535,8 @@ void IouringMgr::PollComplete() req->release_indices_[i]); } } + // Cost must mirror SubmitMergedWrite's AcquireIoWindow exactly. + ReleaseIoWindow(DeviceCmdCost(req->bytes_)); merged_write_req_pool_->Free(req); continue; } @@ -2098,6 +2550,75 @@ void IouringMgr::PollComplete() io_uring_cq_advance(&ring_, cnt); waiting_sqe_.WakeN(cnt); + CHECK_GE(inflight_ios_, cnt); + inflight_ios_ -= cnt; + + if (io_stats_enabled_) + { + if (round_prev_us_ != 0) + { + const uint64_t r = loop_now_us_ - round_prev_us_; + round_sum_us_ += r; + round_max_us_ = std::max(round_max_us_, r); + ++round_cnt_; + } + round_prev_us_ = loop_now_us_; + ++stats_iters_; + if (cnt > 0) + { + ++stats_polls_nonzero_; + stats_cqes_ += cnt; + ++stats_batch_hist_[cnt > 8 ? 0 : cnt]; + } + if (loop_now_us_ >= stats_next_flush_us_) + { + if (stats_next_flush_us_ != 0 && VLOG_IS_ON(1) && stats_cqes_ > 0) + { + const uint64_t n = stage_cnt_ > 0 ? stage_cnt_ : 1; + VLOG(1) << "opstages n=" << stage_cnt_ + << " gate_avg=" << stage_sum_us_[0] / n + << " sqe2cqe_avg=" << stage_sum_us_[1] / n + << " cqe2res_avg=" << stage_sum_us_[2] / n + << " q_wait_avg=" << stage_sum_us_[3] / n + << " start_lag_avg=" << stage_sum_us_[5] / n + << " task2gate_avg=" << stage_sum_us_[4] / n + << " q_wait_max=" << stage_max_us_[3] + << " start_lag_max=" << stage_max_us_[5] + << " task2gate_max=" << stage_max_us_[4] + << " round_avg_ns=" + << (round_cnt_ ? round_sum_us_ * 1000 / round_cnt_ : 0) + << " round_max_us=" << round_max_us_ + << " gate_max=" << stage_max_us_[0] + << " sqe2cqe_max=" << stage_max_us_[1] + << " cqe2res_max=" << stage_max_us_[2]; + std::fill( + std::begin(stage_sum_us_), std::end(stage_sum_us_), 0); + std::fill( + std::begin(stage_max_us_), std::end(stage_max_us_), 0); + stage_cnt_ = 0; + round_sum_us_ = 0; + round_max_us_ = 0; + round_cnt_ = 0; + VLOG(1) << "loopstats tid=" << syscall(SYS_gettid) + << " iters/s=" << stats_iters_ / 5 + << " cqes/s=" << stats_cqes_ / 5 << " avg_batch=" + << static_cast(stats_cqes_) / + static_cast(stats_polls_nonzero_) + << " hist(1..8,9+)=" << stats_batch_hist_[1] << "," + << stats_batch_hist_[2] << "," << stats_batch_hist_[3] + << "," << stats_batch_hist_[4] << "," + << stats_batch_hist_[5] << "," << stats_batch_hist_[6] + << "," << stats_batch_hist_[7] << "," + << stats_batch_hist_[8] << "," << stats_batch_hist_[0]; + } + stats_iters_ = 0; + stats_polls_nonzero_ = 0; + stats_cqes_ = 0; + std::fill( + std::begin(stats_batch_hist_), std::end(stats_batch_hist_), 0); + stats_next_flush_us_ = loop_now_us_ + 5'000'000; + } + } } int IouringMgr::MakeDir(FdIdx dir_fd, const char *path) @@ -2276,13 +2797,26 @@ KvError IouringMgr::FdatasyncFiles(const TableIdent &tbl_id, // Fsync all dirty files/directory. std::vector reqs; reqs.reserve(fds.size()); + // Rate budget (M4): one device op per fsync, no bytes. Other metadata + // ops (open, statx, rename, ...) stay exempt, matching M1. + rate_budget_.Acquire( + static_cast(fds.size()), 0, ThdTask()->IsBackground()); + // Instrumented for IO QoS evaluation (io_qos.md): fsync stalls are a + // distinct interference channel from page-IO queueing. + const uint64_t fsync_start_us = shard->ReadTimeMicroseconds(); for (LruFD::Ref &fd_ref : fds) { // FsyncReq elements have pointer stability, because we have reserved // enough space for this vector so that it will never reallocate. const FsyncReq &req = reqs.emplace_back(ThdTask(), fd_ref); auto [fd, registered] = req.fd_ref_.FdPair(); - io_uring_sqe *sqe = GetSQE(UserDataType::BaseReq, &req); + // Window admission (M2): a flush occupies a device queue slot like + // any command, so a checkpoint's batch must not bypass the cap. + // Per-SQE acquire, like the page-read paths; a mid-loop wait lets + // the shard loop submit the already-charged SQEs. Released once + // per BaseReqFsync CQE in PollComplete. + AcquireIoWindow(1); + io_uring_sqe *sqe = GetSQE(UserDataType::BaseReqFsync, &req); if (registered) { sqe->flags |= IOSQE_FIXED_FILE; @@ -2290,6 +2824,12 @@ KvError IouringMgr::FdatasyncFiles(const TableIdent &tbl_id, io_uring_prep_fsync(sqe, fd, IORING_FSYNC_DATASYNC); } ThdTask()->WaitIo(); + fdatasync_count_.store( + fdatasync_count_.load(std::memory_order_relaxed) + reqs.size(), + std::memory_order_relaxed); + fdatasync_us_.store(fdatasync_us_.load(std::memory_order_relaxed) + + shard->DurationMicroseconds(fsync_start_us), + std::memory_order_relaxed); // Check results. KvError err = KvError::NoError; @@ -3141,6 +3681,7 @@ io_uring_sqe *IouringMgr::GetSQE(UserDataType type, const void *user_ptr) // state does not leak to non-fixed operations. sqe->flags = 0; ThdTask()->inflight_io_++; + ++inflight_ios_; prepared_sqe_++; return sqe; } @@ -4243,7 +4784,7 @@ std::pair CloudStoreMgr::TrimRestoredCacheUsage() bool CloudStoreMgr::IsIdle() { - return file_cleaner_.status_ == TaskStatus::Idle && + return IouringMgr::IsIdle() && file_cleaner_.status_ == TaskStatus::Idle && pending_gc_cleanup_.empty() && active_prewarm_tasks_ == 0 && inflight_cloud_slots_ == 0 && !obj_store_.HasPendingWork(); } diff --git a/src/eloq_store.cpp b/src/eloq_store.cpp index 5f2fbe15..4fa15f1c 100644 --- a/src/eloq_store.cpp +++ b/src/eloq_store.cpp @@ -148,6 +148,14 @@ bool EloqStore::ValidateOptions(KvOptions &opts) LOG(ERROR) << "Option max_global_request_batch cannot be zero"; return false; } + // WriteRateOps divides by this on every write, even with the rate + // limiter disabled; a sub-page quantum would also overcharge writes. + if (opts.rate_limit_io_unit < 4 * KB) + { + LOG(ERROR) << "Option rate_limit_io_unit (" << opts.rate_limit_io_unit + << ") must be at least 4KB"; + return false; + } if ((opts.data_page_size & (page_align - 1)) != 0) { LOG(ERROR) << "Option data_page_size is not page aligned"; @@ -197,11 +205,6 @@ bool EloqStore::ValidateOptions(KvOptions &opts) LOG(ERROR) << "Invalid option overflow_pointers"; return false; } - if (opts.max_write_batch_pages == 0) - { - LOG(ERROR) << "Invalid option max_write_batch_pages"; - return false; - } if (!opts.cloud_store_path.empty()) { LOG(ERROR) << "cloud mode already support standby, reset " @@ -2196,6 +2199,10 @@ bool EloqStore::SendRequest(KvRequest *req) } req->err_ = KvError::NoError; + if (IoStatsEnabled()) + { + req->dbg_enqueue_us_ = Shard::ReadTimeMicroseconds(); + } #ifdef ELOQ_MODULE_ENABLED { std::lock_guard lk(req->mutex_); @@ -2372,6 +2379,12 @@ void EloqStore::InitializeMetrics(metrics::MetricsRegistry *metrics_registry, metrics::Type::Gauge); metrics_meters_[i]->Register(metrics::NAME_ELOQSTORE_LOCAL_SPACE_LIMIT, metrics::Type::Gauge); + // The per-class in-flight page gauges (read / bg-read / write) + // measured the retired M1/M2 count budgets. The M4 rate budget has + // no per-class instantaneous page depth to report (it meters + // cumulative ops/bytes and a class-blind command window), so these + // gauges are not registered; dedicated rate-budget metrics are a + // follow-up (docs/design/io_qos.md). } enable_eloqstore_metrics_ = true; @@ -2451,6 +2464,16 @@ size_t EloqStore::TailScratchAcquireCount(size_t shard_id) const return io_mgr == nullptr ? 0 : io_mgr->TailScratchAcquireCount(); } +IoQosStats EloqStore::GetIoQosStats(size_t shard_id) const +{ + if (shard_id >= shards_.size() || shards_[shard_id] == nullptr) + { + return {}; + } + AsyncIoManager *io_mgr = shards_[shard_id]->IoManager(); + return io_mgr == nullptr ? IoQosStats{} : io_mgr->GetIoQosStats(); +} + bool EloqStore::IsStopped() const { return status_.load(std::memory_order_acquire) == Status::Stopped; diff --git a/src/kv_options.cpp b/src/kv_options.cpp index 96b73b2b..256b6fd2 100644 --- a/src/kv_options.cpp +++ b/src/kv_options.cpp @@ -151,13 +151,75 @@ int KvOptions::LoadFromIni(const char *path) } if (reader.HasValue(sec_run, "max_inflight_write")) { - max_inflight_write = - reader.GetUnsigned(sec_run, "max_inflight_write", 4096); + max_inflight_write = reader.GetUnsigned( + sec_run, "max_inflight_write", max_inflight_write); + } + if (reader.HasValue(sec_run, "max_inflight_read")) + { + max_inflight_read = reader.GetUnsigned(sec_run, "max_inflight_read", 0); + LOG(WARNING) << "max_inflight_read is deprecated and has no effect; " + "device admission control is disk_rate_limit_iops " + "(with rate_bg_ratio) and max_inflight_io"; + } + if (reader.HasValue(sec_run, "bg_read_ratio")) + { + bg_read_ratio = reader.GetUnsigned(sec_run, "bg_read_ratio", 25); + LOG(WARNING) << "bg_read_ratio is deprecated and has no effect; " + "the background share of the device rate budget is " + "rate_bg_ratio"; + } + if (reader.HasValue(sec_run, "disk_rate_limit_iops")) + { + disk_rate_limit_iops = + reader.GetUnsigned64(sec_run, "disk_rate_limit_iops", 275'000); + } + if (reader.HasValue(sec_run, "disk_rate_limit_mbps")) + { + disk_rate_limit_mbps = + reader.GetUnsigned64(sec_run, "disk_rate_limit_mbps", 0); + } + if (reader.HasValue(sec_run, "rate_limit_burst_ms")) + { + // Fall back to the current (member-default) value, not a literal, so + // a malformed entry cannot silently change the policy. + rate_limit_burst_ms = reader.GetUnsigned( + sec_run, "rate_limit_burst_ms", rate_limit_burst_ms); + } + if (reader.HasValue(sec_run, "rate_limit_io_unit")) + { + std::string io_unit_str = reader.Get(sec_run, "rate_limit_io_unit", ""); + const uint64_t parsed = ParseSizeWithUnit(io_unit_str); + // Range check on the wide type before narrowing: e.g. "4GB" is + // nonzero as uint64_t but truncates to uint32_t(0), which would + // reach WriteRateOps' division. Minimum is 4KB (see kv_options.h). + if (parsed < static_cast(4 * KB) || + parsed > std::numeric_limits::max()) + { + LOG(WARNING) << "rate_limit_io_unit '" << io_unit_str + << "' is invalid (minimum 4KB); keeping default " + << rate_limit_io_unit << " bytes"; + } + else + { + rate_limit_io_unit = static_cast(parsed); + } + } + if (reader.HasValue(sec_run, "rate_bg_ratio")) + { + rate_bg_ratio = reader.GetUnsigned(sec_run, "rate_bg_ratio", 25); + } + if (reader.HasValue(sec_run, "max_inflight_io")) + { + max_inflight_io = reader.GetUnsigned(sec_run, "max_inflight_io", 0); } if (reader.HasValue(sec_run, "max_write_batch_pages")) { max_write_batch_pages = reader.GetUnsigned(sec_run, "max_write_batch_pages", 64); + LOG(WARNING) + << "Option max_write_batch_pages is deprecated and has no " + "effect; device write admission is paced by the rate budget " + "(disk_rate_limit_iops, see docs/design/io_qos.md)"; } if (reader.HasValue(sec_run, "coroutine_stack_size")) { @@ -398,6 +460,14 @@ bool KvOptions::operator==(const KvOptions &other) const manifest_limit == other.manifest_limit && fd_limit == other.fd_limit && io_queue_size == other.io_queue_size && max_inflight_write == other.max_inflight_write && + max_inflight_read == other.max_inflight_read && + bg_read_ratio == other.bg_read_ratio && + disk_rate_limit_iops == other.disk_rate_limit_iops && + disk_rate_limit_mbps == other.disk_rate_limit_mbps && + rate_limit_burst_ms == other.rate_limit_burst_ms && + rate_limit_io_unit == other.rate_limit_io_unit && + rate_bg_ratio == other.rate_bg_ratio && + max_inflight_io == other.max_inflight_io && max_write_batch_pages == other.max_write_batch_pages && coroutine_stack_size == other.coroutine_stack_size && num_retained_archives == other.num_retained_archives && diff --git a/src/storage/object_store.cpp b/src/storage/object_store.cpp index 6f623325..c4f8745d 100644 --- a/src/storage/object_store.cpp +++ b/src/storage/object_store.cpp @@ -793,6 +793,10 @@ bool AsyncHttpManager::SetupUploadRequest(ObjectStore::UploadTask *task, "Content-Length: " + std::to_string(task->file_size_); task->headers_ = curl_slist_append(task->headers_, content_length.c_str()); task->headers_ = curl_slist_append(task->headers_, "Expect:"); + // Suppress libcurl's default form-urlencoded Content-Type. It is not + // covered by the presigned-URL signature (SignedHeaders=host), and strict + // validators (e.g. s3proxy) reject the request when it is present. + task->headers_ = curl_slist_append(task->headers_, "Content-Type:"); // Add conditional headers if provided if (!task->if_match_.empty()) diff --git a/src/storage/shard.cpp b/src/storage/shard.cpp index ed01eae8..2dcae50e 100644 --- a/src/storage/shard.cpp +++ b/src/storage/shard.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -121,6 +122,33 @@ void Shard::InitIoMgrAndPagePool() io_mgr_and_page_pool_inited_.store(true, std::memory_order_release); } +#ifdef ELOQSTORE_WITH_TXSERVICE +void Shard::CollectPeriodicGauges(metrics::Meter *meter) +{ + if (++gauge_collection_round_count_ % + metrics::ELOQSTORE_GAUGE_COLLECTION_INTERVAL != + 0) + { + return; + } + + meter->Collect(metrics::NAME_ELOQSTORE_INDEX_BUFFER_POOL_USED, + static_cast(index_mgr_.GetBufferPoolUsed())); + meter->Collect(metrics::NAME_ELOQSTORE_OPEN_FILE_COUNT, + static_cast(io_mgr_->GetOpenFileCount())); + meter->Collect(metrics::NAME_ELOQSTORE_LOCAL_SPACE_USED, + static_cast(io_mgr_->GetLocalSpaceUsed())); + + // The per-class in-flight page gauges belonged to the retired M1/M2 + // count budgets; the M4 rate budget exposes no equivalent per-class + // instantaneous depth, so nothing is collected for them here (they are + // no longer registered). Reporting the class-blind command window under + // a "read pages" name would mislabel writes as reads and read zero + // whenever the window is disabled, so it is deliberately not done. + // Dedicated rate-budget metrics are a follow-up (docs/design/io_qos.md). +} +#endif + void Shard::WorkLoop() { shard = this; @@ -130,7 +158,8 @@ void Shard::WorkLoop() // and no active tasks. // This allows the thread to exit gracefully when the store is stopped. std::array reqs; - auto dequeue_requests = [this, &reqs]() -> int + auto dequeue_requests = [this, + &reqs](uint64_t *queue_wait_us = nullptr) -> int { size_t nreqs = requests_.try_dequeue_bulk(reqs.data(), reqs.size()); // Idle state, wait for new requests or exit. @@ -150,8 +179,14 @@ void Shard::WorkLoop() return 0; } const auto timeout = std::chrono::milliseconds(100); + const uint64_t wait_start = + queue_wait_us == nullptr ? 0 : ReadTimeMicroseconds(); nreqs = requests_.wait_dequeue_bulk_timed( reqs.data(), reqs.size(), timeout); + if (queue_wait_us != nullptr) + { + *queue_wait_us += ReadTimeMicroseconds() - wait_start; + } } return nreqs; @@ -180,21 +215,69 @@ void Shard::WorkLoop() } #endif - io_mgr_->Submit(); - - io_mgr_->PollComplete(); - PromoteReadyDelayedReopenRequests(); - ExecuteReadyTasks(); - - int nreqs = dequeue_requests(); - if (nreqs < 0) + if (!IoStatsEnabled()) { - // Exit. - break; + io_mgr_->Submit(); + io_mgr_->PollComplete(); + PromoteReadyDelayedReopenRequests(); + int nreqs = dequeue_requests(); + if (nreqs < 0) + { + break; + } + for (int i = 0; i < nreqs; i++) + { + OnReceivedReq(reqs[i]); + } + ExecuteReadyTasks(); + io_mgr_->FlushSubmit(); } - for (int i = 0; i < nreqs; i++) + else { - OnReceivedReq(reqs[i]); + // Stats mode: time each loop phase; report any round > 1ms + // with its phase breakdown to locate rare multi-ms stalls. + timespec cpu0; + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &cpu0); + const uint64_t t0 = ReadTimeMicroseconds(); + io_mgr_->Submit(); + const uint64_t t1 = ReadTimeMicroseconds(); + io_mgr_->PollComplete(); + const uint64_t t2 = ReadTimeMicroseconds(); + PromoteReadyDelayedReopenRequests(); + const uint64_t t3 = ReadTimeMicroseconds(); + uint64_t queue_wait_us = 0; + int nreqs = dequeue_requests(&queue_wait_us); + if (nreqs < 0) + { + break; + } + for (int i = 0; i < nreqs; i++) + { + OnReceivedReq(reqs[i]); + } + const uint64_t t4 = ReadTimeMicroseconds(); + ExecuteReadyTasks(); + const uint64_t t5 = ReadTimeMicroseconds(); + io_mgr_->FlushSubmit(); + const uint64_t t6 = ReadTimeMicroseconds(); + const uint64_t total_us = t6 - t0; + const uint64_t active_us = total_us - queue_wait_us; + if (active_us > 1000) + { + timespec cpu1; + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &cpu1); + const uint64_t cpu_us = + (cpu1.tv_sec - cpu0.tv_sec) * 1000000ULL + + (cpu1.tv_nsec - cpu0.tv_nsec) / 1000; + LOG(INFO) << "SLOWROUND total=" << total_us + << "us active=" << active_us << "us cpu=" << cpu_us + << "us submit=" << t1 - t0 << " poll=" << t2 - t1 + << " promote=" << t3 - t2 + << " intake=" << t4 - t3 - queue_wait_us + << " execute=" << t5 - t4 << " flush=" << t6 - t5 + << " queue_wait=" << queue_wait_us + << " nreqs=" << nreqs; + } } #ifdef ELOQSTORE_WITH_TXSERVICE @@ -205,6 +288,7 @@ void Shard::WorkLoop() metrics::NAME_ELOQSTORE_WORK_ONE_ROUND_DURATION, round_start); meter->Collect(metrics::NAME_ELOQSTORE_TASK_MANAGER_ACTIVE_TASKS, static_cast(task_mgr_.NumActive())); + CollectPeriodicGauges(meter); } #endif } @@ -563,6 +647,10 @@ GlobalRegisteredMemory *Shard::GlobalRegMem() void Shard::OnReceivedReq(KvRequest *req) { + if (IoStatsEnabled()) + { + req->dbg_dequeue_us_ = ReadTimeMicroseconds(); + } if (req->Reopen()) { req->SetReopen(false); @@ -1090,6 +1178,10 @@ void Shard::RetryOomRequest(KvRequest *req) #else req->done_.store(false, std::memory_order_relaxed); #endif + if (IoStatsEnabled()) + { + req->dbg_enqueue_us_ = ReadTimeMicroseconds(); + } // AddKvRequest refuses new work once the store is stopping; complete the // retried request with NotRunning instead of dropping it. if (!AddKvRequest(req)) @@ -1308,6 +1400,17 @@ void Shard::WorkOneRound() #endif } + io_mgr_->Submit(); + + io_mgr_->PollComplete(); + PromoteReadyDelayedReopenRequests(); + + // Admit new requests only after PollComplete and Promote, matching + // WorkLoop: everything already in flight is enqueued on ready_tasks_ + // ahead of this round's arrivals, so a burst of new requests cannot + // take precedence over tasks the shard has already started. The + // dequeue itself stays above (is_idle_round depends on nreqs); only + // admission moves. for (size_t i = 0; i < nreqs; ++i) { OnReceivedReq(reqs[i]); @@ -1315,14 +1418,15 @@ void Shard::WorkOneRound() req_queue_size_.fetch_sub(nreqs, std::memory_order_relaxed); - io_mgr_->Submit(); - - io_mgr_->PollComplete(); - PromoteReadyDelayedReopenRequests(); if (DurationMicroseconds(ts_) < FLAGS_max_processing_time_microseconds) { ExecuteReadyTasks(); } + // Issue what this round prepared before handing the thread back to the + // embedding runtime: the next round is an external scheduling decision + // and may be far away, so leaving SQEs for it would idle the device for + // a whole quantum on every IO hop. + io_mgr_->FlushSubmit(); #ifdef ELOQSTORE_WITH_TXSERVICE // Metrics collection: end of round if (store_->EnableMetrics() && !is_idle_round) @@ -1332,32 +1436,7 @@ void Shard::WorkOneRound() round_start); meter->Collect(metrics::NAME_ELOQSTORE_TASK_MANAGER_ACTIVE_TASKS, static_cast(task_mgr_.NumActive())); - - // Increment round counter for frequency-controlled metric collection - work_one_round_count_++; - bool should_collect_gauges = - (work_one_round_count_ % - metrics::ELOQSTORE_GAUGE_COLLECTION_INTERVAL) == 0; - - // Collect used/count metrics (frequency-controlled) - // Note: limit metrics are collected once at initialization in Start() - if (should_collect_gauges) - { - // Collect index buffer pool used - size_t index_used = index_mgr_.GetBufferPoolUsed(); - meter->Collect(metrics::NAME_ELOQSTORE_INDEX_BUFFER_POOL_USED, - static_cast(index_used)); - - // Collect open file count - size_t open_file_count = io_mgr_->GetOpenFileCount(); - meter->Collect(metrics::NAME_ELOQSTORE_OPEN_FILE_COUNT, - static_cast(open_file_count)); - - // Collect local space used - size_t local_space_used = io_mgr_->GetLocalSpaceUsed(); - meter->Collect(metrics::NAME_ELOQSTORE_LOCAL_SPACE_USED, - static_cast(local_space_used)); - } + CollectPeriodicGauges(meter); } #endif } @@ -1448,13 +1527,30 @@ void Shard::InitializeTscFrequency() while (total_slept < MAX_TOTAL_MICROSECONDS) { + // Divide by the MEASURED elapsed wall time, not the + // requested sleep: sleep_for() reliably oversleeps by + // scheduler latency (~60us for a 1ms request), and dividing + // by the nominal duration inflated cycles-per-us by ~6% — + // making every TSC-derived duration (and the M4 rate + // budget's refill) run ~6% slow. The overshoot is + // systematic, so the stability check below cannot catch it. + timespec mono_start, mono_end; + clock_gettime(CLOCK_MONOTONIC, &mono_start); uint64_t start_cycles = __rdtsc(); std::this_thread::sleep_for( std::chrono::microseconds(SLEEP_MICROSECONDS)); uint64_t end_cycles = __rdtsc(); + clock_gettime(CLOCK_MONOTONIC, &mono_end); uint64_t elapsed_cycles = end_cycles - start_cycles; - uint64_t freq = elapsed_cycles / - SLEEP_MICROSECONDS; // cycles per microsecond + const int64_t elapsed_ns = + (mono_end.tv_sec - mono_start.tv_sec) * 1'000'000'000LL + + (mono_end.tv_nsec - mono_start.tv_nsec); + const uint64_t elapsed_us = + static_cast(std::max(elapsed_ns, 0)) / + 1000; + uint64_t freq = + elapsed_cycles / + std::max(elapsed_us, 1); // cycles per us total_slept += SLEEP_MICROSECONDS; diff --git a/src/tasks/read_task.cpp b/src/tasks/read_task.cpp index 7e5d7153..4a3ba37c 100644 --- a/src/tasks/read_task.cpp +++ b/src/tasks/read_task.cpp @@ -14,6 +14,16 @@ namespace eloqstore { namespace { +void BeginReadIoTiming() +{ + if (IoStatsEnabled()) + { + KvTask *task = ThdTask(); + task->op_cqe_us_ = 0; + task->op_start_us_ = Shard::ReadTimeMicroseconds(); + } +} + // Common path shared by all three Read() overloads: descend through the index // tree to the leaf data page that should contain @p search_key, load it, and // position @p iter at the matching entry. Returns KvError::NotFound when the @@ -35,6 +45,7 @@ KvError LocateAndProcess(const TableIdent &tbl_id, uint64_t &expire_ts, Handler &&handler) { + BeginReadIoTiming(); auto [root_handle, err] = shard->IndexManager()->FindRoot(tbl_id); CHECK_KV_ERR(err); RootMeta *meta = root_handle.Get(); @@ -198,6 +209,7 @@ KvError ReadTask::Floor(const TableIdent &tbl_id, uint64_t &expire_ts, IoStringBuffer *large_value) { + BeginReadIoTiming(); auto [root_handle, err] = shard->IndexManager()->FindRoot(tbl_id); CHECK_KV_ERR(err); RootMeta *meta = root_handle.Get(); diff --git a/src/tasks/task.cpp b/src/tasks/task.cpp index 1a33cfd8..682d6644 100644 --- a/src/tasks/task.cpp +++ b/src/tasks/task.cpp @@ -505,9 +505,10 @@ void WaitingZone::WakeOne() } } -void WaitingZone::WakeN(size_t n) +size_t WaitingZone::WakeN(size_t n) { - for (size_t i = 0; i < n; i++) + size_t woken = 0; + while (woken < n) { KvTask *task = PopFront(); if (task == nullptr) @@ -516,7 +517,9 @@ void WaitingZone::WakeN(size_t n) } assert(task->status_ == TaskStatus::Blocked); task->Resume(); + woken++; } + return woken; } void WaitingZone::WakeAll() diff --git a/src/tasks/write_task.cpp b/src/tasks/write_task.cpp index 35e92e13..1ad70124 100644 --- a/src/tasks/write_task.cpp +++ b/src/tasks/write_task.cpp @@ -302,16 +302,15 @@ KvError WriteTask::WritePage(VarPage page, FilePageId file_page_id) KvError err = IoMgr()->WritePage(tbl_ident_, std::move(page), file_page_id); CHECK_KV_ERR(err); - if (inflight_io_ >= opts->max_write_batch_pages) - { - // Avoid long running WriteTask block ReadTask/ScanTask - err = WaitWrite(); - CHECK_KV_ERR(err); - } - else - { - YieldToLowPQ(); - } + // In-flight write IO is bounded by the shard's write budget + // (max_inflight_write, enforced inside IoMgr()->WritePage; see + // docs/design/io_qos.md M1), which replaced the per-task + // max_write_batch_pages drain-to-zero throttle here: budget admission + // keeps a steady in-flight level instead of a sawtooth, and unlike the + // per-task cap it also counts compaction and concurrent write tasks. + // Completion errors are collected by the terminal WaitWrite() before + // UpdateMeta/SyncData. The yield below is CPU-side cooperation only. + YieldToLowPQ(); return KvError::NoError; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7262e7e9..5d42a39d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -45,6 +45,7 @@ set(UTEST_SOURCES iouring_tail_scratch.cpp segment_allocator.cpp data_page_cache.cpp + io_qos.cpp ) string( REPLACE ".cpp" "" BASENAMES_UTEST "${UTEST_SOURCES}" ) diff --git a/tests/batch_write.cpp b/tests/batch_write.cpp index e6981387..7c2c0d35 100644 --- a/tests/batch_write.cpp +++ b/tests/batch_write.cpp @@ -129,7 +129,6 @@ TEST_CASE("batch write abort releases pinned index pages", opts.num_threads = 1; opts.data_page_size = 4096; opts.buffer_pool_size = 4096; // Allow only a single MemCachedPage. - opts.max_write_batch_pages = 4; opts.auto_oom_retry_times = 0; auto build_entries = @@ -190,7 +189,6 @@ TEST_CASE("batch write task pool handles many partitions concurrently", opts.store_path = {test_path}; opts.num_threads = 1; // single shard, many partitions opts.buffer_pool_size = 4096 * 400; // enough for many pages - opts.max_write_batch_pages = 8; auto make_entries = [](uint32_t base, uint32_t count) { diff --git a/tests/cloud.cpp b/tests/cloud.cpp index 05ef8f19..62da7c01 100644 --- a/tests/cloud.cpp +++ b/tests/cloud.cpp @@ -447,12 +447,10 @@ TEST_CASE("cloud reuse cache enforces budgets across restarts", CleanupStore(options); - auto store = std::make_unique(options); - REQUIRE(store->Start(eloqstore::MainBranchName, 0) == - eloqstore::KvError::NoError); + eloqstore::EloqStore *store = InitStore(options, /*cleanup=*/false); eloqstore::TableIdent tbl_id{"reuse-cache", 0}; - MapVerifier writer(tbl_id, store.get()); + MapVerifier writer(tbl_id, store); writer.SetAutoClean(false); writer.SetAutoValidate(false); writer.SetValueSize(64 << 10); @@ -475,7 +473,7 @@ TEST_CASE("cloud reuse cache enforces budgets across restarts", store->Stop(); REQUIRE(store->Start(eloqstore::MainBranchName, 0) == eloqstore::KvError::NoError); - writer.SetStore(store.get()); + writer.SetStore(store); WriteBatches(writer, next_key, entries_per_batch, batches_per_phase); @@ -489,11 +487,11 @@ TEST_CASE("cloud reuse cache enforces budgets across restarts", store->Stop(); // Tighten the budget to 20MB and verify restore trims/existing files and - // future writes respect the new limit. + // future writes respect the new limit. InitStore (cleanup=false) replaces + // the previous instance while keeping the cached files on disk. options.local_space_limit = 20ULL << 20; - auto trimmed_store = std::make_unique(options); - REQUIRE(trimmed_store->Start("main", 0) == eloqstore::KvError::NoError); - writer.SetStore(trimmed_store.get()); + eloqstore::EloqStore *trimmed_store = InitStore(options, /*cleanup=*/false); + writer.SetStore(trimmed_store); WriteBatches(writer, next_key, entries_per_batch, batches_per_phase / 2); @@ -580,8 +578,7 @@ TEST_CASE("cloud startup restore removes empty idle partition directories", CleanupStore(options); - auto store = std::make_unique(options); - REQUIRE(store->Start() == eloqstore::KvError::NoError); + eloqstore::EloqStore *store = InitStore(options, /*cleanup=*/false); store->Stop(); const eloqstore::TableIdent tbl_id{"reuse_empty_idle", 0}; @@ -617,8 +614,7 @@ TEST_CASE("cloud startup restore removes partitions cleaned to empty", CleanupStore(options); - auto store = std::make_unique(options); - REQUIRE(store->Start() == eloqstore::KvError::NoError); + eloqstore::EloqStore *store = InitStore(options, /*cleanup=*/false); store->Stop(); const eloqstore::TableIdent tbl_id{"reuse_cleanup_idle", 0}; diff --git a/tests/common.cpp b/tests/common.cpp index d51c54eb..28839910 100644 --- a/tests/common.cpp +++ b/tests/common.cpp @@ -8,13 +8,22 @@ #include "utils.h" -eloqstore::EloqStore *InitStore(const eloqstore::KvOptions &opts) +eloqstore::EloqStore *InitStore(const eloqstore::KvOptions &opts, bool cleanup) { static std::unique_ptr eloq_store = nullptr; // Tear down any prior store before constructing the new one, so the old // destructor's worker-thread joins and LRU-cached fd releases finish // before we count the new store's fd budget below. + // + // Tests using this shared fixture must go through InitStore — never mix a + // directly-owned store with this process-global instance. The global + // Options()/Comp() plumbing assumes compatible live stores; overlapping + // incompatible instances can leave teardown reading a nulled/foreign + // global (observed as a flaky SIGSEGV in Prewarmer::Shutdown). Intentional + // multi-instance/topology tests own and coordinate all instances instead. + // Tests that only need to preserve on-disk/cloud state across generations + // (warm restart, cache-trim) pass cleanup = false. if (eloq_store) { if (!eloq_store->IsStopped()) @@ -23,11 +32,14 @@ eloqstore::EloqStore *InitStore(const eloqstore::KvOptions &opts) } eloq_store.reset(); } - if (!opts.cloud_store_path.empty()) + if (cleanup) { - S3TestClient s3_client(opts); + if (!opts.cloud_store_path.empty()) + { + S3TestClient s3_client(opts); + } + CleanupStore(opts); } - CleanupStore(opts); // EloqStore::Start() counts the *process-wide* `/proc/self/fd` and // subtracts it from `fd_limit`. When multiple test cases run in the diff --git a/tests/common.h b/tests/common.h index c5456787..363e5f7f 100644 --- a/tests/common.h +++ b/tests/common.h @@ -78,7 +78,18 @@ const eloqstore::KvOptions cloud_archive_opts = { .pages_per_file_shift = 8, .data_append_mode = true, }; -eloqstore::EloqStore *InitStore(const eloqstore::KvOptions &opts); +/** + * Create (or replace) the per-binary test store. Stops and destroys any + * previously created store first, so at most one EloqStore instance is ever + * started in the process — required by the process-global Options() pointer. + * Tests using this shared fixture must create stores through InitStore. + * Intentional multi-instance/topology tests may own EloqStore instances + * directly and are responsible for compatible process-global options. Pass + * cleanup = false to keep existing local/cloud state (warm-restart and + * cache-reuse tests). + */ +eloqstore::EloqStore *InitStore(const eloqstore::KvOptions &opts, + bool cleanup = true); bool ValidateFileSizes(const eloqstore::KvOptions &opts); diff --git a/tests/data_page_cache.cpp b/tests/data_page_cache.cpp index 06905a97..16f7a70d 100644 --- a/tests/data_page_cache.cpp +++ b/tests/data_page_cache.cpp @@ -178,6 +178,13 @@ TEST_CASE("data page cache: OOM retry under concurrent writes", opts.store_path = {"/tmp/eloqstore"}; opts.num_threads = 1; opts.buffer_pool_size = 128 * 4096; // ~512 KB; comfortably fits one task + // The pool above is too small for a write-buffer pool, so these appends + // take the non-append WritePage path, where write-promotion pins on + // cached pages persist until the write IO completes. Bound the shard's + // in-flight writes so concurrent tasks' pins can't crowd the 128-slot + // pool past what 5 OOM retries can absorb (the deprecated per-task + // max_write_batch_pages drain used to provide this bound implicitly). + opts.max_inflight_write = 32; eloqstore::EloqStore *store = InitStore(opts); @@ -237,7 +244,11 @@ TEST_CASE("data page cache: OOM abort releases pinned pages", // Two slots: just enough for a small steady state, but the OOM batch // below intentionally exceeds it. opts.buffer_pool_size = 2 * 4096; - opts.max_write_batch_pages = 4; + // Keep at most one write in flight so the first batch's write-promotion + // pins release before the next allocation needs a slot. (This test was + // originally tuned to the deprecated max_write_batch_pages drain; the + // write budget expresses the same bound without the drain-to-zero.) + opts.max_inflight_write = 1; opts.auto_oom_retry_times = 0; auto build_entries = diff --git a/tests/eloq_store_test.cpp b/tests/eloq_store_test.cpp index 391143a4..0a9a199d 100644 --- a/tests/eloq_store_test.cpp +++ b/tests/eloq_store_test.cpp @@ -14,6 +14,8 @@ namespace fs = std::filesystem; +static_assert(sizeof(eloqstore::IoQosStats) > 0); + eloqstore::KvOptions CreateValidOptions(const fs::path &test_dir) { eloqstore::KvOptions options; @@ -29,7 +31,8 @@ eloqstore::KvOptions CreateValidOptions(const fs::path &test_dir) fs::path CreateTestDir(const std::string &suffix = "") { - fs::path test_dir = fs::temp_directory_path() / ("eloqstore_test" + suffix); + fs::path test_dir = + fs::temp_directory_path() / "test-data" / ("eloqstore_test" + suffix); fs::create_directories(test_dir); return test_dir; } @@ -41,6 +44,68 @@ void CleanupTestDir(const fs::path &test_dir) fs::remove_all(test_dir); } } + +TEST_CASE("KvOptions parses QoS knobs and preserves malformed defaults", + "[eloq_store]") +{ + REQUIRE(eloqstore::KvOptions{}.max_inflight_write == 32768); + + const fs::path test_dir = CreateTestDir("_qos_options"); + const fs::path ini_path = test_dir / "eloqstore.ini"; + { + std::ofstream ini(ini_path); + REQUIRE(ini.is_open()); + ini << "[run]\nmax_inflight_write = 73\n" + "max_inflight_read = 17\n" + "bg_read_ratio = 42\n" + "[permanent]\nstore_path = /tmp/unused\n"; + } + + eloqstore::KvOptions options; + REQUIRE(options.LoadFromIni(ini_path.c_str()) == 0); + REQUIRE(options.max_inflight_write == 73); + REQUIRE(options.max_inflight_read == 17); + REQUIRE(options.bg_read_ratio == 42); + + { + std::ofstream ini(ini_path); + REQUIRE(ini.is_open()); + ini << "[run]\nmax_inflight_write = invalid\n" + "[permanent]\nstore_path = /tmp/unused\n"; + } + options = eloqstore::KvOptions{}; + REQUIRE(options.LoadFromIni(ini_path.c_str()) == 0); + REQUIRE(options.max_inflight_write == + eloqstore::KvOptions{}.max_inflight_write); + + // rate_limit_io_unit: a valid size within [4KB, UINT32_MAX] is taken; + // below the 4KB minimum or above uint32 range ("4GB" is nonzero as + // uint64_t but would truncate to 0) keeps the default. + { + std::ofstream ini(ini_path); + REQUIRE(ini.is_open()); + ini << "[run]\nrate_limit_io_unit = 8KB\n" + "[permanent]\nstore_path = /tmp/unused\n"; + } + options = eloqstore::KvOptions{}; + REQUIRE(options.LoadFromIni(ini_path.c_str()) == 0); + REQUIRE(options.rate_limit_io_unit == 8 * 1024); + for (const char *bad : {"2KB", "4GB", "0"}) + { + { + std::ofstream ini(ini_path); + REQUIRE(ini.is_open()); + ini << "[run]\nrate_limit_io_unit = " << bad + << "\n[permanent]\nstore_path = /tmp/unused\n"; + } + options = eloqstore::KvOptions{}; + REQUIRE(options.LoadFromIni(ini_path.c_str()) == 0); + REQUIRE(options.rate_limit_io_unit == + eloqstore::KvOptions{}.rate_limit_io_unit); + } + CleanupTestDir(test_dir); +} + TEST_CASE("EloqStore ValidateOptions validates all parameters", "[eloq_store]") { auto test_dir = CreateTestDir("_validate_options"); @@ -49,6 +114,23 @@ TEST_CASE("EloqStore ValidateOptions validates all parameters", "[eloq_store]") // Test valid configuration REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == true); + // Write budget 0 is not a disable switch: every write must be bounded. + options.max_inflight_write = 0; + REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == false); + options = CreateValidOptions(test_dir); // restore valid value + + // rate_limit_io_unit floors at 4KB. WriteRateOps divides by it on + // every write — even with the rate limiter disabled — so zero and + // sub-page quanta must be rejected for programmatically constructed + // options, not just on the INI path. + options.rate_limit_io_unit = 0; + REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == false); + options.rate_limit_io_unit = 2 * 1024; + REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == false); + options.rate_limit_io_unit = 8 * 1024; + REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == true); + options = CreateValidOptions(test_dir); // restore valid value + // Test data_page_size that is not page-aligned options.data_page_size = 4097; // not page-aligned REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == false); @@ -64,9 +146,9 @@ TEST_CASE("EloqStore ValidateOptions validates all parameters", "[eloq_store]") REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == false); options = CreateValidOptions(test_dir); // restore valid value - // Test invalid max_write_batch_pages + // The retired per-task write throttle no longer constrains validation. options.max_write_batch_pages = 0; - REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == false); + REQUIRE(eloqstore::EloqStore::ValidateOptions(options) == true); options = CreateValidOptions(test_dir); // restore valid value // Test invalid max_cloud_concurrency (cloud mode) @@ -253,8 +335,7 @@ TEST_CASE("EloqStore Start validates local store paths", "[eloq_store]") } // test the non exist path - fs::path nonexistent_path = - fs::temp_directory_path() / "nonexistent_eloqstore_test"; + fs::path nonexistent_path = test_dir / "nonexistent_eloqstore_test"; options.store_path = {nonexistent_path}; { eloqstore::EloqStore store(options); @@ -266,7 +347,7 @@ TEST_CASE("EloqStore Start validates local store paths", "[eloq_store]") } // the path is file - fs::path file_path = fs::temp_directory_path() / "eloqstore_file_test"; + fs::path file_path = test_dir / "eloqstore_file_test"; std::ofstream file(file_path); file.close(); options.store_path = {file_path}; diff --git a/tests/io_qos.cpp b/tests/io_qos.cpp new file mode 100644 index 00000000..8cdd89ab --- /dev/null +++ b/tests/io_qos.cpp @@ -0,0 +1,453 @@ +/** + * IO QoS (docs/design/io_qos.md) — M4 device rate budget and in-flight + * command window tests. + * + * Rates and windows are set deliberately tiny so the blocking paths are + * hot, then the tests assert: pacing completes without deadlock (debt + * admission, refill-driven wakes), classes are charged correctly, + * borrowing is foreground-only, disabled mechanisms stay zero-cost, and + * shutdown drains queued waiters cleanly. + */ +#include + +#include +#include +#include +#include +#include + +#include "async_io_manager.h" +#include "common.h" +#include "fail_point.h" +#include "test_utils.h" + +using test_util::MapVerifier; + +namespace eloqstore +{ +DECLARE_uint64(max_processing_time_microseconds); +} + +namespace +{ +eloqstore::IoQosStats ShardStats(const eloqstore::EloqStore *store) +{ + return store->GetIoQosStats(0); +} +} // namespace + +TEST_CASE("defaults: rate limiting on, unit workload never blocks", "[io_qos]") +{ + // Rate limiting is on by default (275K IOPS per disk / shard here): + // it must charge a trivial single-threaded workload without ever + // blocking it, and the io window is off by default. + eloqstore::EloqStore *store = InitStore(default_opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(200); + verify.WriteRnd(0, 2000, 0, 25); + for (int i = 0; i < 100; i++) + { + verify.Read(std::rand() % 2000); + } + + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.rate_.admitted_ops_ > 0); + REQUIRE(stats.rate_.blocked_count_ == 0); + REQUIRE(stats.io_window_blocked_ == 0); + REQUIRE(stats.io_window_hwm_ == 0); +} + +TEST_CASE("rate budget: iops = 0 disables it, stats stay zero", "[io_qos]") +{ + // Rate limiting is ON by default (275K IOPS per disk, a cloud-NVMe + // starting point); zero must disable it entirely. + REQUIRE(eloqstore::KvOptions{}.disk_rate_limit_iops == 275'000); + eloqstore::KvOptions opts = default_opts; + opts.disk_rate_limit_iops = 0; + eloqstore::EloqStore *store = InitStore(opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(200); + verify.WriteRnd(0, 1000, 0, 25); + for (int i = 0; i < 50; i++) + { + verify.Read(std::rand() % 1000); + } + + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.rate_.admitted_ops_ == 0); + REQUIRE(stats.rate_.blocked_count_ == 0); + REQUIRE(stats.bg_rate_.blocked_count_ == 0); +} + +TEST_CASE("rate budget: charges device IO and completes under a tiny rate", + "[io_qos]") +{ + // A deliberately low rate (well below what the workload wants) must + // pace the run without deadlock or error: debt admission guarantees + // progress for costs larger than the bucket, and refill-driven wakes + // guarantee waiter progress. Blocked counters must move, every op + // must be accounted, and balances must drain (no in-flight concept — + // admitted ops are cumulative). + eloqstore::KvOptions opts = default_opts; + opts.disk_rate_limit_iops = 2000; // per "disk"; one path, one shard + opts.rate_limit_burst_ms = 4; + eloqstore::EloqStore *store = InitStore(opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(200); + verify.WriteRnd(0, 500, 0, 25); + for (int i = 0; i < 50; i++) + { + verify.Read(std::rand() % 500); + } + + eloqstore::IoQosStats stats = ShardStats(store); + // The budget is partitioned per class: rate_ meters foreground reads, + // bg_rate_ meters background IO (background reads and all write-path + // IO). Both classes did device IO here, so both must show charges. + REQUIRE(stats.rate_.admitted_ops_ > 0); + REQUIRE(stats.rate_.admitted_bytes_ > 0); + REQUIRE(stats.bg_rate_.admitted_ops_ > 0); + REQUIRE(stats.bg_rate_.admitted_bytes_ > 0); +} + +TEST_CASE("rate budget: foreground borrows background's idle surplus", + "[io_qos]") +{ + // Reads run after all writes complete, so the background class is idle + // and its share must be lent to foreground. A single 600KB overflow + // value read issues ~150 page reads concurrently (one ReadPages batch), + // so foreground demand instantaneously exceeds its whole burst of ops + // and the surplus must be granted from the idle background bucket — + // deterministic regardless of device speed (a rate-based sequential + // read stream is not: it can stay under the foreground share). Every + // borrowed op is still accounted to the foreground (borrower) class. + eloqstore::KvOptions opts = default_opts; + opts.disk_rate_limit_iops = 2000; + opts.rate_limit_burst_ms = 4; + opts.overflow_pointers = 128; + eloqstore::EloqStore *store = InitStore(opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(600 * 1024); + verify.Upsert(1); + verify.Upsert(2); + // Background is idle now; these foreground reads each fan out into a + // large concurrent page-read batch that outruns the foreground burst. + verify.Read(1); + verify.Read(2); + + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.rate_.admitted_ops_ > 0); + REQUIRE(stats.rate_.borrowed_ops_ > 0); + // Borrowed ops are a subset of the class's admitted ops. + REQUIRE(stats.rate_.borrowed_ops_ <= stats.rate_.admitted_ops_); + // Borrowing is asymmetric: background must never borrow foreground's + // share (symmetric borrowing collapsed storm isolation — see + // RateBudget::CanAdmit). + REQUIRE(stats.bg_rate_.borrowed_ops_ == 0); +} + +TEST_CASE("rate budget: overflow read batch far above one burst of tokens", + "[io_qos]") +{ + // 600KB values span ~150 overflow pages; GetOverflowValue issues + // 128-page ReadPages batches. At 2000 units/s with a 4 ms bucket + // (8 banked tokens) a batch is far above one burst, so per-page + // acquisition must pace it through debt and refills without deadlock. + eloqstore::KvOptions opts = default_opts; + opts.disk_rate_limit_iops = 2000; + opts.rate_limit_burst_ms = 4; + opts.overflow_pointers = 128; + eloqstore::EloqStore *store = InitStore(opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(600 * 1024); + verify.Upsert(1); + verify.Upsert(2); + verify.Read(1); + verify.Read(2); + + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.rate_.admitted_ops_ > 0); + REQUIRE(stats.rate_.blocked_count_ > 0); +} + +TEST_CASE("rate budget: shutdown while tasks queue behind the gate", "[io_qos]") +{ + // Several overflow reads contend for a tiny rate, then the store is + // stopped while they are still queued at the gate. Stop must drain + // cleanly (refill-driven wakes depend only on the shard loop running) + // and every request must complete. + eloqstore::KvOptions opts = default_opts; + opts.disk_rate_limit_iops = 2000; + opts.rate_limit_burst_ms = 4; + opts.overflow_pointers = 128; + eloqstore::EloqStore *store = InitStore(opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(600 * 1024); + verify.Upsert(0, 4); + + std::array reqs; + std::array keys; + std::atomic done{0}; + for (uint32_t i = 0; i < reqs.size(); i++) + { + keys[i] = test_util::Key(i, 7); // MapVerifier's key format + reqs[i].SetArgs(test_tbl_id, keys[i]); + store->ExecAsyn(&reqs[i], + 0, + [&done](eloqstore::KvRequest *) + { done.fetch_add(1, std::memory_order_relaxed); }); + } + store->Stop(); // blocks until the shard drains + + REQUIRE(done.load() == 4); + for (auto &req : reqs) + { + REQUIRE(req.Error() == eloqstore::KvError::NoError); + } +} + +TEST_CASE("io window: class-blind device-command cap bounds and completes", + "[io_qos]") +{ + // A tiny window must bound in-flight device commands while everything + // still completes. 600KB values make GetOverflowValue issue 128-page + // ReadPages batches — far above a 2-command window — so per-command + // acquisition must block and make progress. No rate budget: the + // window must work standalone. (Point writes/reads alone cannot + // contend a per-shard window: one partition serializes its write + // task and the verifier reads synchronously.) + eloqstore::KvOptions opts = default_opts; + opts.disk_rate_limit_iops = 0; // window standalone, rate off + opts.max_inflight_io = 2; + opts.overflow_pointers = 128; + eloqstore::EloqStore *store = InitStore(opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(600 * 1024); + verify.Upsert(1); + verify.Upsert(2); + verify.Read(1); + verify.Read(2); + + eloqstore::IoQosStats stats = ShardStats(store); + // The window drains to zero once all requests have completed. + REQUIRE(stats.io_window_inflight_ == 0); + // A 128-page batch through a 2-command window must have waited. + REQUIRE(stats.io_window_blocked_ > 0); + // Page IO respects the cap strictly (cost-1 commands); only a merged + // write's oversized admission (ceil(1MB/256KB) = 4 > 2, admitted + // alone) may exceed it. + REQUIRE(stats.io_window_hwm_ >= 2); + REQUIRE(stats.io_window_hwm_ <= 4); +} + +TEST_CASE("io window: negative KvTaskPageRead CQE releases and recovers", + "[io_qos]") +{ + // Error-path release symmetry: a failed read CQE must still release + // its window command, the failure must surface as IoFail, and the + // store must keep serving afterwards. (Ported from the count-budget + // negative-CQE tests; the rate budget has no completion-time release + // by design, so the window carries this invariant now.) + eloqstore::KvOptions opts = default_opts; + opts.disk_rate_limit_iops = 0; + opts.max_inflight_io = 4; + eloqstore::EloqStore *store = InitStore(opts); + const eloqstore::TableIdent tbl_id{"qos-read-cqe", 0}; + MapVerifier seed(tbl_id, store, false); + seed.SetValueSize(200); + seed.Upsert(0, 100); + const std::string key = test_util::Key(7, 7); + const std::string expected = seed.DataSet().at(key).value_; + + eloqstore::ReadRequest failed; + failed.SetArgs(tbl_id, key); + eloqstore::FailPoint::GetInstance().ArmOnce("KvTaskPageReadCqe"); + store->ExecSync(&failed); + eloqstore::FailPoint::GetInstance().Disarm(); + + REQUIRE(failed.Error() == eloqstore::KvError::IoFail); + REQUIRE(ShardStats(store).io_window_inflight_ == 0); + + eloqstore::ReadRequest recovery; + recovery.SetArgs(tbl_id, key); + store->ExecSync(&recovery); + REQUIRE(recovery.Error() == eloqstore::KvError::NoError); + REQUIRE(recovery.value_ == expected); + REQUIRE(ShardStats(store).io_window_inflight_ == 0); +} + +TEST_CASE("io window: negative MergedWriteReq CQE releases and recovers", + "[io_qos]") +{ + // Same invariant for the merged-write cost (ceil(len/256KB) commands): + // acquire and release must stay symmetric on the error path. + eloqstore::KvOptions opts = append_opts; + opts.disk_rate_limit_iops = 0; + opts.max_inflight_io = 8; + eloqstore::EloqStore *store = InitStore(opts); + const eloqstore::TableIdent tbl_id{"qos-merged-cqe", 0}; + const uint64_t ts = utils::UnixTs(); + + eloqstore::BatchWriteRequest failed; + failed.SetTableId(tbl_id); + for (uint32_t i = 0; i < 400; ++i) + { + failed.AddWrite(test_util::Key(i, 7), + std::string(3000, 'm'), + ts, + eloqstore::WriteOp::Upsert); + } + eloqstore::FailPoint::GetInstance().ArmOnce("MergedWriteReqCqe"); + store->ExecSync(&failed); + eloqstore::FailPoint::GetInstance().Disarm(); + + REQUIRE(failed.Error() == eloqstore::KvError::IoFail); + REQUIRE(ShardStats(store).io_window_inflight_ == 0); + + eloqstore::BatchWriteRequest recovery; + recovery.SetTableId(tbl_id); + recovery.AddWrite("recovery", "value", ts + 1, eloqstore::WriteOp::Upsert); + store->ExecSync(&recovery); + REQUIRE(recovery.Error() == eloqstore::KvError::NoError); + REQUIRE(ShardStats(store).io_window_inflight_ == 0); +} + +TEST_CASE("io window: negative fsync CQE releases and recovers", "[io_qos]") +{ + // FdatasyncFiles charges one window command per fsync SQE + // (BaseReqFsync) so a checkpoint's flush batch cannot bypass + // max_inflight_io. Acquire and release must stay symmetric on the + // error path: the failed CQE still releases its command (in Debug the + // ReleaseIoWindow underflow assert would fire if the acquire were + // missing), the failure surfaces on the write request, and the store + // keeps serving afterwards. + eloqstore::KvOptions opts = default_opts; + opts.disk_rate_limit_iops = 0; + opts.max_inflight_io = 2; + eloqstore::EloqStore *store = InitStore(opts); + const eloqstore::TableIdent tbl_id{"qos-fsync-cqe", 0}; + const uint64_t ts = utils::UnixTs(); + + eloqstore::BatchWriteRequest failed; + failed.SetTableId(tbl_id); + failed.AddWrite("fsync-key", "value", ts, eloqstore::WriteOp::Upsert); + eloqstore::FailPoint::GetInstance().ArmOnce("BaseReqFsyncCqe"); + store->ExecSync(&failed); + eloqstore::FailPoint::GetInstance().Disarm(); + + REQUIRE(failed.Error() == eloqstore::KvError::IoFail); + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.fdatasync_count_ > 0); + REQUIRE(stats.io_window_inflight_ == 0); + + eloqstore::BatchWriteRequest recovery; + recovery.SetTableId(tbl_id); + recovery.AddWrite("recovery", "value", ts + 1, eloqstore::WriteOp::Upsert); + store->ExecSync(&recovery); + REQUIRE(recovery.Error() == eloqstore::KvError::NoError); + REQUIRE(ShardStats(store).io_window_inflight_ == 0); +} + +TEST_CASE("io qos stats: concurrent sampling", "[io_qos][stats]") +{ + // IoQosStats must be safely sampleable from another thread while the + // shard runs (observability counters are relaxed atomics; the shard + // thread is the only writer). Ported from the count-budget suite. + eloqstore::KvOptions opts = default_opts; + opts.disk_rate_limit_iops = 20000; + opts.max_inflight_io = 8; + eloqstore::EloqStore *store = InitStore(opts); + + std::atomic stop{false}; + std::atomic samples{0}; + std::atomic workload_active{false}; + std::atomic active_samples{0}; + std::thread sampler( + [&] + { + while (!stop.load(std::memory_order_relaxed)) + { + const eloqstore::IoQosStats stats = store->GetIoQosStats(0); + (void) stats; + samples.fetch_add(1, std::memory_order_relaxed); + if (workload_active.load(std::memory_order_relaxed)) + { + active_samples.fetch_add(1, std::memory_order_relaxed); + } + } + }); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(3000); + workload_active.store(true, std::memory_order_relaxed); + for (int round = 0; round < 3; ++round) + { + verify.WriteRnd(0, 1000, 0, 50); + for (int i = 0; i < 100; ++i) + { + verify.Read(std::rand() % 1000); + } + } + workload_active.store(false, std::memory_order_relaxed); + + stop.store(true, std::memory_order_relaxed); + sampler.join(); + const eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(samples.load(std::memory_order_relaxed) > 0); + REQUIRE(active_samples.load(std::memory_order_relaxed) > 0); + REQUIRE(stats.rate_.admitted_ops_ > 0); + REQUIRE(stats.io_window_inflight_ == 0); +} + +TEST_CASE("rate budget: write ops cost is ceil(bytes / io_unit)", "[io_qos]") +{ + // The write ops charge is the single WriteRateOps formula used by both + // WritePage and SubmitMergedWrite. At the 4KB default quantum a data + // page costs 1 op and a 1MB merged write costs 256; a finer 2KB unit + // doubles both. Locks the currency so the SubmitMergedWrite clamp + // regression (which floored the unit at data_page_size) cannot return. + using eloqstore::IouringMgr; + REQUIRE(IouringMgr::WriteRateOpsFor(4 * 1024, 4 * 1024) == 1); + REQUIRE(IouringMgr::WriteRateOpsFor(1u << 20, 4 * 1024) == 256); + REQUIRE(IouringMgr::WriteRateOpsFor(4 * 1024, 2 * 1024) == 2); + REQUIRE(IouringMgr::WriteRateOpsFor(1u << 20, 2 * 1024) == 512); + // Partial final unit rounds up. + REQUIRE(IouringMgr::WriteRateOpsFor(4 * 1024 + 1, 4 * 1024) == 2); +} + +TEST_CASE("rate budget: bytes bucket alone paces merged writes", "[io_qos]") +{ + // Only the bytes bucket enabled: append_opts inherits the nonzero + // default disk_rate_limit_iops, so it must be explicitly zeroed or this + // would enable both buckets and never exercise the ops-disabled branch + // of Positive() / the ops-not-debited path in Charge(). The tiny byte + // budget must actually block background writes. + eloqstore::KvOptions opts = append_opts; + opts.disk_rate_limit_iops = 0; + opts.disk_rate_limit_mbps = 8; + opts.rate_limit_burst_ms = 4; + eloqstore::EloqStore *store = InitStore(opts); + + MapVerifier verify(test_tbl_id, store, false); + verify.SetValueSize(1000); + verify.WriteRnd(0, 1000, 0, 25); + for (int i = 0; i < 20; i++) + { + verify.Read(std::rand() % 1000); + } + + eloqstore::IoQosStats stats = ShardStats(store); + REQUIRE(stats.bg_rate_.admitted_bytes_ > 0); + // The small byte budget must have paced (blocked) background writes. + // (Ops are still counted in admitted_ops_ for observability even with + // the ops dimension disabled; only the ops *balance* is left untouched, + // which is what the overflow fix guards — not asserted here.) + REQUIRE(stats.bg_rate_.blocked_count_ > 0); +} diff --git a/tests/large_value_benchmark.cpp b/tests/large_value_benchmark.cpp index 8823c091..df7a7547 100644 --- a/tests/large_value_benchmark.cpp +++ b/tests/large_value_benchmark.cpp @@ -1,7 +1,9 @@ // tests/large_value_benchmark.cpp // // Informational benchmarks for the zero-copy large-value read/write paths. -// Excluded from the normal (non-benchmark) test run. +// Excluded from the normal test run via the Catch2 hidden tag "[.]": not +// discovered by ctest/CI and skipped on a bare binary run; select the tag +// explicitly to execute. // // Run with: // sudo prlimit --memlock=unlimited ./large_value_benchmark -t '[benchmark]' @@ -477,7 +479,7 @@ constexpr ConcurrencyConfig kConcurrencyConfigs[] = { // inference server with multiple in-flight requests) would see. // ============================================================================ TEST_CASE("Benchmark: concurrent random reads (large-value vs overflow-page)", - "[benchmark][read-latency]") + "[.][benchmark][read-latency]") { constexpr size_t kTargetDataset = 256ULL * 1024 * 1024; constexpr int kMinKeys = 256; @@ -661,7 +663,7 @@ TEST_CASE("Benchmark: concurrent random reads (large-value vs overflow-page)", // std::string path writes overflow pages via regular io_uring writes. // ============================================================================ TEST_CASE("Benchmark: batch-write throughput IoStringBuffer vs std::string", - "[benchmark][write-throughput]") + "[.][benchmark][write-throughput]") { BenchHarness h; eloqstore::KvOptions opts = MakeBenchOpts(h); @@ -834,7 +836,7 @@ TEST_CASE("Benchmark: batch-write throughput IoStringBuffer vs std::string", // Expected: smaller yield_every → more frequent yields → lower p99 overhead. // ============================================================================ TEST_CASE("Benchmark: foreground read p99 during segment compaction", - "[benchmark][compact-overhead]") + "[.][benchmark][compact-overhead]") { const uint32_t yield_values[] = {1, 8, 32}; @@ -1222,7 +1224,7 @@ std::vector TimePinnedReadsOverloadC(eloqstore::EloqStore *store, // throughput; the kernel just reads the same source range M times. // ============================================================================ TEST_CASE("Benchmark: pinned-mode batch-write throughput", - "[benchmark][write-throughput][pinned]") + "[.][benchmark][write-throughput][pinned]") { struct SizeCase { @@ -1429,7 +1431,7 @@ TEST_CASE("Benchmark: pinned-mode batch-write throughput", // scratch acquire/release. // ============================================================================ TEST_CASE("Benchmark: pinned-write tail-scratch overhead", - "[benchmark][write-latency][pinned][tail-scratch]") + "[.][benchmark][write-latency][pinned][tail-scratch]") { struct SizeCase { @@ -1545,7 +1547,7 @@ TEST_CASE("Benchmark: pinned-write tail-scratch overhead", // aggregate MB/s (total bytes / wall time). // ============================================================================ TEST_CASE("Benchmark: pinned-mode concurrent random reads", - "[benchmark][read-latency][pinned]") + "[.][benchmark][read-latency][pinned]") { constexpr size_t kTargetDataset = 256ULL * 1024 * 1024; constexpr int kMinKeys = 256; @@ -1677,7 +1679,7 @@ TEST_CASE("Benchmark: pinned-mode concurrent random reads", // window saves the index navigation on the (W-1)/W subsequent reads. // ============================================================================ TEST_CASE("Benchmark: pinned-mode concurrent windowed-locality reads", - "[benchmark][read-latency][pinned]") + "[.][benchmark][read-latency][pinned]") { constexpr size_t kTargetDataset = 256ULL * 1024 * 1024; constexpr int kMinKeys = 256;