From 52efbdf53bc7b4dc12ab17aee6928cb4c6b1cbbb Mon Sep 17 00:00:00 2001 From: lokax Date: Thu, 26 Feb 2026 14:49:47 +0800 Subject: [PATCH 01/38] ckpt --- store_handler/data_store_service_client.cpp | 3 +- .../eloq_store_data_store.cpp | 39 +- tx_service/include/cc/cc_request.h | 52 +- tx_service/include/cc/local_cc_shards.h | 32 +- tx_service/include/cc/template_cc_map.h | 171 +-- tx_service/src/cc/local_cc_shards.cpp | 1000 ++++++++++------- 6 files changed, 673 insertions(+), 624 deletions(-) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index 233864e0..10052f23 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -318,7 +318,6 @@ bool DataStoreServiceClient::PutAll( DLOG(INFO) << "DataStoreServiceClient::PutAll called with " << flush_task.size() << " tables to flush."; uint64_t now = txservice::LocalCcShards::ClockTsInMillseconds(); - // Create global coordinator SyncPutAllData *sync_putall = sync_putall_data_pool_.NextObject(); PoolableGuard sync_putall_guard(sync_putall); @@ -490,6 +489,7 @@ bool DataStoreServiceClient::PutAll( callback_data->Clear(); callback_data->Free(); } + return false; } } @@ -505,6 +505,7 @@ bool DataStoreServiceClient::PutAll( metrics::kv_meter->Collect( metrics::NAME_KV_FLUSH_ROWS_TOTAL, records_count, "base"); } + return true; } diff --git a/store_handler/eloq_data_store_service/eloq_store_data_store.cpp b/store_handler/eloq_data_store_service/eloq_store_data_store.cpp index fe796c7a..e6b49ca5 100644 --- a/store_handler/eloq_data_store_service/eloq_store_data_store.cpp +++ b/store_handler/eloq_data_store_service/eloq_store_data_store.cpp @@ -59,6 +59,12 @@ inline void BuildKey(const WriteRecordsRequest &write_req, uint16_t key_parts, std::string &key_out) { + size_t total_size = 0; + for (uint16_t i = 0; i < key_parts; ++i) + { + total_size += write_req.GetKeyPart(key_first_idx + i).size(); + } + key_out.reserve(key_out.size() + total_size); size_t part_idx = key_first_idx; for (uint16_t i = 0; i < key_parts; ++i, ++part_idx) { @@ -72,6 +78,12 @@ inline void BuildValue(const WriteRecordsRequest &write_req, uint16_t rec_parts, std::string &rec_out) { + size_t total_size = 0; + for (uint16_t i = 0; i < rec_parts; ++i) + { + total_size += write_req.GetRecordPart(rec_first_idx + i).size(); + } + rec_out.reserve(rec_out.size() + total_size); size_t part_idx = rec_first_idx; for (uint16_t i = 0; i < rec_parts; ++i, ++part_idx) { @@ -214,26 +226,28 @@ void EloqStoreDataStore::BatchWriteRecords(WriteRecordsRequest *write_req) ::eloqstore::BatchWriteRequest &kv_write_req = write_op->EloqStoreRequest(); - std::vector<::eloqstore::WriteDataEntry> entries; - size_t rec_cnt = write_req->RecordsCount(); - entries.reserve(rec_cnt); + const size_t rec_cnt = write_req->RecordsCount(); const uint16_t parts_per_key = write_req->PartsCountPerKey(); const uint16_t parts_per_record = write_req->PartsCountPerRecord(); - size_t first_idx = 0; + + std::vector<::eloqstore::WriteDataEntry> entries(rec_cnt); + size_t key_offset = 0; + size_t val_offset = 0; + for (size_t i = 0; i < rec_cnt; ++i) { - ::eloqstore::WriteDataEntry entry; - first_idx = i * parts_per_key; - BuildKey(*write_req, first_idx, parts_per_key, entry.key_); - first_idx = i * parts_per_record; - BuildValue(*write_req, first_idx, parts_per_record, entry.val_); + ::eloqstore::WriteDataEntry &entry = entries[i]; + BuildKey(*write_req, key_offset, parts_per_key, entry.key_); + BuildValue(*write_req, val_offset, parts_per_record, entry.val_); + key_offset += parts_per_key; + val_offset += parts_per_record; + entry.timestamp_ = write_req->GetRecordTs(i); entry.op_ = (write_req->KeyOpType(i) == WriteOpType::PUT ? ::eloqstore::WriteOp::Upsert : ::eloqstore::WriteOp::Delete); - uint64_t ttl = write_req->GetRecordTtl(i); - entry.expire_ts_ = ttl == UINT64_MAX ? 0 : ttl; - entries.emplace_back(std::move(entry)); + const uint64_t ttl = write_req->GetRecordTtl(i); + entry.expire_ts_ = (ttl == UINT64_MAX) ? 0u : ttl; } if (!std::ranges::is_sorted( @@ -250,6 +264,7 @@ void EloqStoreDataStore::BatchWriteRecords(WriteRecordsRequest *write_req) kv_write_req.SetArgs(eloq_store_table_id, std::move(entries)); uint64_t user_data = reinterpret_cast(write_op); + if (!eloq_store_service_->ExecAsyn(&kv_write_req, user_data, OnBatchWrite)) { LOG(ERROR) << "Send write request to EloqStore failed for table: " diff --git a/tx_service/include/cc/cc_request.h b/tx_service/include/cc/cc_request.h index 47ebfce8..388663c9 100644 --- a/tx_service/include/cc/cc_request.h +++ b/tx_service/include/cc/cc_request.h @@ -8865,11 +8865,7 @@ struct ScanSliceDeltaSizeCcForRangePartition : public CcRequestBase struct ScanDeltaSizeCcForHashPartition : public CcRequestBase { - static constexpr size_t ScanBatchSize = 128; - ScanDeltaSizeCcForHashPartition(const TableName &table_name, - uint64_t last_datasync_ts, - uint64_t scan_ts, uint64_t ng_id, int64_t ng_term, uint64_t txn, @@ -8877,8 +8873,6 @@ struct ScanDeltaSizeCcForHashPartition : public CcRequestBase : table_name_(table_name), node_group_id_(ng_id), node_group_term_(ng_term), - last_datasync_ts_(last_datasync_ts), - scan_ts_(scan_ts), schema_version_(schema_version) { tx_number_ = txn; @@ -8970,44 +8964,27 @@ struct ScanDeltaSizeCcForHashPartition : public CcRequestBase return node_group_term_; } - uint64_t LastDataSyncTs() const - { - return last_datasync_ts_; - } - - uint64_t ScanTs() const - { - return scan_ts_; - } - void AbortCcRequest(CcErrorCode err_code) override { assert(err_code != CcErrorCode::NO_ERROR); SetError(err_code); } - TxKey &PausedKey() + void SetKeyCounts(const size_t data_key_count, const size_t dirty_key_count) { - return pause_key_; - } - - void UpdateKeyCount(const bool need_ckpt) - { - ++scanned_key_count_; - if (need_ckpt) - { - ++updated_key_count_; - } + data_key_count_ = data_key_count; + dirty_key_count_ = dirty_key_count; } size_t UpdatedMemory() const { - const uint64_t scanned = scanned_key_count_; - if (scanned == 0) + if (data_key_count_ == 0) return 0; + // Use cc map's data_key_count_ and dirty_key_count_ for estimation. // integer math with rounding up to avoid systematic underestimation return static_cast( - (updated_key_count_ * memory_usage_ + scanned - 1) / scanned); + (dirty_key_count_ * memory_usage_ + data_key_count_ - 1) / + data_key_count_); } void SetMemoryUsage(const uint64_t memory) @@ -9024,20 +9001,11 @@ struct ScanDeltaSizeCcForHashPartition : public CcRequestBase const TableName &table_name_; uint32_t node_group_id_; int64_t node_group_term_; - // It is used as a hint to decide if a page has dirty data since last round - // of checkpoint. It is guaranteed that all entries committed before this ts - // are synced into data store. - uint64_t last_datasync_ts_; - // Target ts. Collect all data changes committed before this ts into data - // sync vec. - uint64_t scan_ts_; - // Position that we left off during last round of scan. - TxKey pause_key_; uint64_t schema_version_; - // Number of keys scanned / updated, and per-core memory usage. - uint64_t scanned_key_count_{0}; - uint64_t updated_key_count_{0}; + // From cc map: total data keys and dirty keys needing checkpoint. + size_t data_key_count_{0}; + size_t dirty_key_count_{0}; uint64_t memory_usage_{0}; CcErrorCode err_{CcErrorCode::NO_ERROR}; diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index 316d8042..0fc899da 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -2476,6 +2476,13 @@ class LocalCcShards uint32_t range_cnt, int32_t &out_next_partition_id); + /** + * @brief Map a data_sync_worker index to the fixed flush_data_worker index. + * Used when data_sync_worker count != flush_data_worker count so that each + * data_sync_worker consistently targets one flush_data_worker. + */ + size_t DataSyncWorkerToFlushDataWorker(size_t data_sync_worker_id) const; + /** * @brief Add a flush task entry to the flush task. If the there's no * pending flush task, create a new flush task and add the entry to it. @@ -2487,16 +2494,21 @@ class LocalCcShards WorkerThreadContext flush_data_worker_ctx_; - // The flush task that has not reached the max pending flush size. - // New flush task entry will be added to this buffer. This task will - // be appended to pending_flush_work_ when it reaches the max pending flush - // size, which will then be processed by flush data worker. - FlushDataTask cur_flush_buffer_; - // Flush task queue for flush data worker to process. - std::deque> pending_flush_work_; - - void FlushDataWorker(); - void FlushData(std::unique_lock &flush_worker_lk); + // Per-worker flush buffers. Each DataSyncWorker has its own buffer. + // New flush task entry will be added to the corresponding buffer. This task + // will be appended to pending_flush_work_[worker_idx] when it reaches the + // max pending flush size, which will then be processed by the corresponding + // flush data worker. Store as pointers because FlushDataTask contains a + // bthread::Mutex and is non-movable/non-copyable, which cannot be stored + // directly in a vector that may reallocate. + std::vector> cur_flush_buffers_; + // Per-worker flush task queues. Each FlushDataWorker processes its + // corresponding queue. + std::vector>> pending_flush_work_; + + void FlushDataWorker(size_t worker_idx); + void FlushData(std::unique_lock &flush_worker_lk, + size_t worker_idx); // Memory controller for data sync. DataSyncMemoryController data_sync_mem_controller_; diff --git a/tx_service/include/cc/template_cc_map.h b/tx_service/include/cc/template_cc_map.h index 1870cbe5..6d69eb0f 100644 --- a/tx_service/include/cc/template_cc_map.h +++ b/tx_service/include/cc/template_cc_map.h @@ -7981,132 +7981,11 @@ class TemplateCcMap : public CcMap return false; } - auto &paused_key = req.PausedKey(); - - auto deduce_iterator = [this](const KeyT &search_key) -> Iterator - { - Iterator it; - std::pair search_pair = - ForwardScanStart(search_key, true); - it = search_pair.first; - if (search_pair.second == ScanType::ScanGap) - { - ++it; - } - return it; - }; - - auto next_page_it = [this](Iterator &end_it) -> Iterator - { - Iterator it = end_it; - if (it != End()) - { - CcPage *ccp = - it.GetPage(); - assert(ccp != nullptr); - if (ccp->next_page_ == PagePosInf()) - { - it = End(); - } - else - { - it = Iterator(ccp->next_page_, 0, &neg_inf_); - } - } - return it; - }; - - Iterator key_it; - Iterator req_end_it; - - // The key iterator. - const KeyT *const search_start_key = - paused_key.GetKey() == nullptr ? KeyT::NegativeInfinity() - : paused_key.GetKey(); - key_it = deduce_iterator(*search_start_key); - - // The request end iterator - req_end_it = deduce_iterator(*KeyT::PositiveInfinity()); - - // Since we might skip the page that end_it is on if it's not updated - // since last ckpt, it might skip end_it. If the last page is skipped it - // will be set as the first entry on the next page. Also check if - // (key_it == end_next_page_it). - Iterator req_end_next_page_it = next_page_it(req_end_it); - - // The current slice end iterator - Iterator slice_end_it = req_end_it; - Iterator slice_end_next_page_it = req_end_next_page_it; - - // ScanSliceDeltaSizeCcForRangePartition is running on TxProcessor - // thread. To avoid blocking other transaction for a long time, we only - // process ScanBatchSize number of keys in each round. - for (size_t scan_cnt = 0; - scan_cnt < ScanDeltaSizeCcForHashPartition::ScanBatchSize && - key_it != req_end_it && key_it != req_end_next_page_it; - ++scan_cnt) - { - CcEntry *cce = - key_it->second; - CcPage *ccp = - key_it.GetPage(); - assert(ccp); - - if (ccp->last_dirty_commit_ts_ <= req.LastDataSyncTs()) - { - // Skip the pages that have no updates since last data sync. - if (ccp->next_page_ == PagePosInf()) - { - key_it = End(); - } - else - { - key_it = Iterator(ccp->next_page_, 0, &neg_inf_); - } - - // Check the slice iterator. - if (key_it == slice_end_it || key_it == slice_end_next_page_it) - { - // Reset the slice end iterator - slice_end_it = req_end_it; - slice_end_next_page_it = next_page_it(slice_end_it); - } - continue; - } - - const uint64_t commit_ts = cce->CommitTs(); - - // The commit_ts <= 1 means the key is non-existed or a new inserted - // key that the tx has not finished post-processing. - req.UpdateKeyCount(cce->NeedCkpt() && commit_ts > 1 && - commit_ts <= req.ScanTs()); - - // Forward key iterator - ++key_it; - - if (key_it == slice_end_it) - { - // Update the end it. - slice_end_it = req_end_it; - slice_end_next_page_it = next_page_it(slice_end_it); - } - } - - if (key_it == req_end_it || key_it == req_end_next_page_it) - { - int64_t allocated, committed; - mi_thread_stats(&allocated, &committed); - req.SetMemoryUsage(allocated); - req.SetFinish(); - } - else - { - paused_key = key_it->first->CloneTxKey(); - shard_->EnqueueLowPriorityCcRequest(&req); - } - - // Access ScanSliceDeltaSizeCcForRangePartition member variable is - // unsafe after SetFinished(). + req.SetKeyCounts(data_key_count_, dirty_data_key_count_); + int64_t allocated, committed; + mi_thread_stats(&allocated, &committed); + req.SetMemoryUsage(allocated); + req.SetFinish(); return false; } @@ -8243,10 +8122,35 @@ class TemplateCcMap : public CcMap return true; } + void AdjustDataKeyStats(int64_t size_delta, int64_t dirty_delta) + { + if (table_name_.IsMeta()) + { + return; + } + + if (size_delta != 0) + { + assert(size_delta >= 0 || + data_key_count_ >= static_cast(-size_delta)); + data_key_count_ = static_cast( + static_cast(data_key_count_) + size_delta); + } + + if (dirty_delta != 0) + { + assert(dirty_delta >= 0 || + dirty_data_key_count_ >= static_cast(-dirty_delta)); + dirty_data_key_count_ = static_cast( + static_cast(dirty_data_key_count_) + dirty_delta); + } + } + void OnEntryFlushed(bool was_dirty, bool is_persistent) override { if (was_dirty && is_persistent) { + AdjustDataKeyStats(0, -1); shard_->AdjustDataKeyStats(table_name_, 0, -1); } } @@ -8266,6 +8170,7 @@ class TemplateCcMap : public CcMap page_it = ccmp_.find(ccpage->FirstKey()); } const bool was_dirty = cce->IsDirty(); + AdjustDataKeyStats(-1, was_dirty ? -1 : 0); shard_->AdjustDataKeyStats(table_name_, -1, was_dirty ? -1 : 0); ccpage->Remove(cce); @@ -8342,6 +8247,8 @@ class TemplateCcMap : public CcMap if (total_freed > 0 || dirty_freed > 0) { + AdjustDataKeyStats(-static_cast(total_freed), + -static_cast(dirty_freed)); shard_->AdjustDataKeyStats(table_name_, -static_cast(total_freed), -static_cast(dirty_freed)); @@ -8383,6 +8290,8 @@ class TemplateCcMap : public CcMap cce->ClearLocks(*shard_, cc_ng_id_); } + AdjustDataKeyStats(-static_cast(page->Size()), + -static_cast(dirty_freed)); shard_->AdjustDataKeyStats(table_name_, -static_cast(page->Size()), -static_cast(dirty_freed)); @@ -8800,6 +8709,7 @@ class TemplateCcMap : public CcMap { if (!was_dirty && cce->IsDirty()) { + AdjustDataKeyStats(0, +1); shard_->AdjustDataKeyStats(table_name_, 0, +1); } } @@ -9671,6 +9581,7 @@ class TemplateCcMap : public CcMap { normal_obj_sz_ += normal_rec_change; } + AdjustDataKeyStats(+(end_idx - first_index), 0); shard_->AdjustDataKeyStats( table_name_, +(end_idx - first_index), 0); @@ -9709,6 +9620,7 @@ class TemplateCcMap : public CcMap assert(new_key_cnt > 0); // Update CCMap size + AdjustDataKeyStats(+new_key_cnt, 0); shard_->AdjustDataKeyStats(table_name_, +new_key_cnt, 0); } else @@ -9802,6 +9714,7 @@ class TemplateCcMap : public CcMap assert(new_key_cnt > 0); // Update Ccmap size + AdjustDataKeyStats(+new_key_cnt, 0); shard_->AdjustDataKeyStats(table_name_, +new_key_cnt, 0); } @@ -10081,6 +9994,7 @@ class TemplateCcMap : public CcMap // update lru list shard_->UpdateLruList(target_page, true); + AdjustDataKeyStats(+1, 0); shard_->AdjustDataKeyStats(table_name_, +1, 0); return Iterator(target_page, idx_in_page, &neg_inf_); @@ -11254,6 +11168,9 @@ class TemplateCcMap : public CcMap { normal_obj_sz_ -= clean_guard->CleanObjectCount(); } + AdjustDataKeyStats( + -static_cast(clean_guard->FreedCount()), + -static_cast(clean_guard->DirtyFreedCount())); shard_->AdjustDataKeyStats( table_name_, -static_cast(clean_guard->FreedCount()), @@ -11713,6 +11630,8 @@ class TemplateCcMap : public CcMap TemplateCcMapSamplePool *sample_pool_; size_t normal_obj_sz_{ 0}; // The count of all normal status objects, only used for redis + size_t data_key_count_{0}; + size_t dirty_data_key_count_{0}; }; template diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 280719e4..df856953 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -140,10 +140,8 @@ LocalCcShards::LocalCcShards( flush_data_worker_ctx_( std::min(static_cast(conf.at("core_num")), 10)), #endif - - cur_flush_buffer_( - static_cast(MB(conf.at("node_memory_limit_mb")) * - (FLAGS_ckpt_buffer_ratio - 0.025))), + // cur_flush_buffers_ will be resized in StartBackgroudWorkers() + // when worker_num_ is determined data_sync_mem_controller_(static_cast( MB(conf.at("node_memory_limit_mb")) * FLAGS_ckpt_buffer_ratio)), statistics_worker_ctx_(1), @@ -291,11 +289,28 @@ LocalCcShards::~LocalCcShards() void LocalCcShards::StartBackgroudWorkers() { + // Ensure FlushDataWorker count matches DataSyncWorker count + // Note: worker_num_ is const, so we can't modify it. We use + // data_sync_worker_ctx_.worker_num_ to initialize the vectors and assert + // they match. + // Initialize per-worker data structures + const size_t worker_num = flush_data_worker_ctx_.worker_num_; + // Calculate buffer size + const uint64_t buffer_size = + data_sync_mem_controller_.FlushMemoryQuota() / (worker_num); + cur_flush_buffers_.clear(); + for (size_t i = 0; i < worker_num; ++i) + { + cur_flush_buffers_.emplace_back( + std::make_unique(buffer_size)); + } + pending_flush_work_.resize(worker_num); + // Starts flush worker threads firstly. for (int id = 0; id < flush_data_worker_ctx_.worker_num_; id++) { std::thread &thd = flush_data_worker_ctx_.worker_thd_.emplace_back( - [this] { FlushDataWorker(); }); + [this, id] { FlushDataWorker(id); }); std::string thread_name = "flush_data_" + std::to_string(id); pthread_setname_np(thd.native_handle(), thread_name.c_str()); } @@ -4715,16 +4730,58 @@ void LocalCcShards::DataSyncForHashPartition( assert(table_name.Type() == TableType::Primary); + ScanDeltaSizeCcForHashPartition scan_delta_size_cc( + table_name, + ng_id, + ng_term, + data_sync_txm->TxNumber(), + catalog_rec.Schema()->Version()); + + EnqueueToCcShard(worker_idx, &scan_delta_size_cc); + scan_delta_size_cc.Wait(); + + if (scan_delta_size_cc.IsError()) + { + LOG(ERROR) << "DataSync scan slice delta size failed on table " + << table_name.StringView() << " with error code: " + << static_cast(scan_delta_size_cc.ErrorCode()); + + AbortTx(data_sync_txm); + std::lock_guard task_worker_lk(data_sync_worker_ctx_.mux_); + data_sync_task_queue_[worker_idx].emplace_front(data_sync_task); + data_sync_worker_ctx_.cv_.notify_one(); + return; + } + auto core_number = cc_shards_.size(); - auto approximate_partition_number_this_node = std::max( - 1U, total_hash_partitions / Sharder::Instance().NodeGroupCount()); - auto approximate_partition_number_this_core = - std::max(1UL, approximate_partition_number_this_node / core_number); - const size_t flush_buffer_size = cur_flush_buffer_.GetFlushBufferSize(); - constexpr size_t default_data_file_size = 8ULL * 1024 * 1024; - const size_t scan_concurrency = - flush_buffer_size / - (default_data_file_size * approximate_partition_number_this_core); + const size_t partition_number = total_hash_partitions; + auto partition_number_this_core = + partition_number / core_number + + (worker_idx < partition_number % core_number); + std::vector partition_ids; + partition_ids.reserve(partition_number_this_core); + for (size_t i = 0; i < partition_number_this_core; ++i) + { + partition_ids.emplace_back(worker_idx + core_number * i); + } + assert(partition_number_this_core == partition_ids.size()); + const auto updated_memory = scan_delta_size_cc.UpdatedMemory(); + auto updated_memory_per_partition = + partition_number_this_core ? updated_memory / partition_number_this_core + : 0; + const size_t flush_buffer_size = + cur_flush_buffers_[DataSyncWorkerToFlushDataWorker(static_cast( + data_sync_task->id_ % + data_sync_worker_ctx_.worker_num_))] + ->GetFlushBufferSize(); + + const size_t partition_number_per_scan = + std::max(1UL, + updated_memory_per_partition != 0 + ? (flush_buffer_size / updated_memory_per_partition) + : partition_number_this_core); + const size_t scan_concurrency = core_number; + if (scan_concurrency > 0) { bool need_notify = scan_concurrency > @@ -4744,432 +4801,416 @@ void LocalCcShards::DataSyncForHashPartition( data_sync_task->flight_task_cnt_ += 1; } - HashPartitionDataSyncScanCc scan_cc(table_name, - data_sync_task->data_sync_ts_, - ng_id, - ng_term, - DATA_SYNC_SCAN_BATCH_SIZE, - data_sync_txm->TxNumber(), - data_sync_task->forward_cache_, - last_sync_ts, - data_sync_task->filter_lambda_, - catalog_rec.Schema()->Version()); - bool scan_data_drained = false; - - auto data_sync_vec = std::make_unique>(); - auto archive_vec = std::make_unique>(); - auto mv_base_vec = - std::make_unique>>(); - uint64_t vec_mem_usage = 0; + for (size_t i = 0; i < partition_number_this_core; + i += partition_number_per_scan) + { + size_t min_partition_id_this_scan = partition_ids[i]; + size_t max_partition_id_this_scan = + partition_ids[std::min(i + partition_number_per_scan, + partition_number_this_core) - + 1]; + std::function filter_lambda = + [min_partition_id_this_scan, + max_partition_id_this_scan, + &filter_func = + data_sync_task->filter_lambda_](const size_t hash_code) + { + return (hash_code % total_hash_partitions) >= + min_partition_id_this_scan && + (hash_code % total_hash_partitions) <= + max_partition_id_this_scan && + (!filter_func || filter_func(hash_code)); + }; - // Note: `DataSyncScanCc` needs to ensure that no two ckpt_rec with the - // same Key can be generated. Our subsequent algorithms are based on - // this assumption. + HashPartitionDataSyncScanCc scan_cc(table_name, + data_sync_task->data_sync_ts_, + ng_id, + ng_term, + DATA_SYNC_SCAN_BATCH_SIZE, + data_sync_txm->TxNumber(), + data_sync_task->forward_cache_, + last_sync_ts, + filter_lambda, + catalog_rec.Schema()->Version()); + bool scan_data_drained = false; + + auto data_sync_vec = std::make_unique>(); + auto archive_vec = std::make_unique>(); + auto mv_base_vec = + std::make_unique>>(); + uint64_t vec_mem_usage = 0; - assert(worker_idx < cc_shards_.size()); + // Note: `DataSyncScanCc` needs to ensure that no two ckpt_rec with the + // same Key can be generated. Our subsequent algorithms are based on + // this assumption. - while (!scan_data_drained) - { - EnqueueLowPriorityCcRequestToShard(worker_idx, &scan_cc); - scan_cc.Wait(); + assert(worker_idx < cc_shards_.size()); - if (scan_cc.IsError() && - scan_cc.ErrorCode() != CcErrorCode::LOG_NOT_TRUNCATABLE) + while (!scan_data_drained) { - LOG(ERROR) << "DataSync scan failed on table " - << table_name.StringView() << " with error code: " - << static_cast(scan_cc.ErrorCode()); + EnqueueLowPriorityCcRequestToShard(worker_idx, &scan_cc); + scan_cc.Wait(); - PostProcessHashPartitionDataSyncTask( - std::move(data_sync_task), - data_sync_txm, - DataSyncTask::CkptErrorCode::SCAN_ERROR); - - return; - } - if (scan_cc.ErrorCode() == CcErrorCode::LOG_NOT_TRUNCATABLE) - { - data_sync_task->status_->SetEntriesSkippedAndNoTruncateLog(); - } - - scan_data_drained = true; - - // Send cache to target node group if needed. - if (data_sync_task->forward_cache_) - { - std::shared_lock meta_lk(meta_data_mux_); - const auto bucket_infos = GetAllBucketInfosNoLocking(ng_id); - if (bucket_infos == nullptr) + if (scan_cc.IsError() && + scan_cc.ErrorCode() != CcErrorCode::LOG_NOT_TRUNCATABLE) { - // no longer node group owner, abort the task - LOG(ERROR) << "DataSync: Failed to get bucket infos for " - "ng#" - << ng_id; + LOG(ERROR) << "DataSync scan failed on table " + << table_name.StringView() << " with error code: " + << static_cast(scan_cc.ErrorCode()); + PostProcessHashPartitionDataSyncTask( std::move(data_sync_task), data_sync_txm, DataSyncTask::CkptErrorCode::SCAN_ERROR); + return; } + if (scan_cc.ErrorCode() == CcErrorCode::LOG_NOT_TRUNCATABLE) + { + data_sync_task->status_->SetEntriesSkippedAndNoTruncateLog(); + } + + scan_data_drained = true; - std::unordered_map - send_cache_closures; - for (size_t idx = 0; idx < scan_cc.accumulated_scan_cnt_; idx++) + // Send cache to target node group if needed. + if (data_sync_task->forward_cache_) { - FlushRecord &ref = scan_cc.DataSyncVec()[idx]; - uint16_t bucket_id = - Sharder::MapKeyHashToBucketId(ref.Key().Hash()); - NodeGroupId dest_ng = - bucket_infos->at(bucket_id)->DirtyBucketOwner(); - assert(dest_ng != UINT32_MAX); + std::shared_lock meta_lk(meta_data_mux_); + const auto bucket_infos = GetAllBucketInfosNoLocking(ng_id); + if (bucket_infos == nullptr) + { + // no longer node group owner, abort the task + LOG(ERROR) << "DataSync: Failed to get bucket infos for " + "ng#" + << ng_id; + PostProcessHashPartitionDataSyncTask( + std::move(data_sync_task), + data_sync_txm, + DataSyncTask::CkptErrorCode::SCAN_ERROR); + return; + } - // Put the record into the request for this node group. - auto ins_res = send_cache_closures.try_emplace(dest_ng); - remote::UploadBatchRequest *req_ptr = nullptr; - if (ins_res.second) + std::unordered_map + send_cache_closures; + for (size_t idx = 0; idx < scan_cc.accumulated_scan_cnt_; idx++) { - uint32_t node_id = - Sharder::Instance().LeaderNodeId(dest_ng); - std::shared_ptr channel = - Sharder::Instance().GetCcNodeServiceChannel(node_id); - if (channel == nullptr) + FlushRecord &ref = scan_cc.DataSyncVec()[idx]; + uint16_t bucket_id = + Sharder::MapKeyHashToBucketId(ref.Key().Hash()); + NodeGroupId dest_ng = + bucket_infos->at(bucket_id)->DirtyBucketOwner(); + assert(dest_ng != UINT32_MAX); + + // Put the record into the request for this node group. + auto ins_res = send_cache_closures.try_emplace(dest_ng); + remote::UploadBatchRequest *req_ptr = nullptr; + if (ins_res.second) { - // Fail to establish the channel to the tx node. - // Just skip the cache sending since it is a best - // effort try to performance improvement. - LOG(ERROR) << "UploadBatch: Failed to init the " - "channel of ng#" - << dest_ng; - send_cache_closures.erase(ins_res.first); + uint32_t node_id = + Sharder::Instance().LeaderNodeId(dest_ng); + std::shared_ptr channel = + Sharder::Instance().GetCcNodeServiceChannel( + node_id); + if (channel == nullptr) + { + // Fail to establish the channel to the tx node. + // Just skip the cache sending since it is a best + // effort try to performance improvement. + LOG(ERROR) << "UploadBatch: Failed to init the " + "channel of ng#" + << dest_ng; + send_cache_closures.erase(ins_res.first); + } + else + { + // Create a closure for the first time. + UploadBatchClosure *upload_batch_closure = + new UploadBatchClosure( + [this, data_sync_task, data_sync_txm]( + CcErrorCode res_code, + int32_t dest_ng_term) + { + // We don't care if + // the cache send + // was succeed or + // not since it's a + // best effort + // move. Just pass in no error + // so that it won't cause data sync + // failure. + PostProcessHashPartitionDataSyncTask( + std::move(data_sync_task), + data_sync_txm, + DataSyncTask::CkptErrorCode:: + NO_ERROR); + }, + 10000, + false); + + upload_batch_closure->SetChannel(node_id, channel); + + ins_res.first->second = upload_batch_closure; + req_ptr = + upload_batch_closure->UploadBatchRequest(); + req_ptr->set_node_group_id(dest_ng); + req_ptr->set_node_group_term(-1); + req_ptr->set_table_name_str(table_name.String()); + req_ptr->set_table_type( + remote::ToRemoteType::ConvertTableType( + table_name.Type())); + req_ptr->set_table_engine( + remote::ToRemoteType::ConvertTableEngine( + table_name.Engine())); + req_ptr->set_kind( + remote::UploadBatchKind::DIRTY_BUCKET_DATA); + req_ptr->set_batch_size(0); + // keys + req_ptr->clear_keys(); + // records + req_ptr->clear_records(); + // commit_ts + req_ptr->clear_commit_ts(); + // rec_status + req_ptr->clear_rec_status(); + } } else { - // Create a closure for the first time. - UploadBatchClosure *upload_batch_closure = - new UploadBatchClosure( - [this, data_sync_task, data_sync_txm]( - CcErrorCode res_code, int32_t dest_ng_term) - { - // We don't care if - // the cache send - // was succeed or - // not since it's a - // best effort - // move. Just pass in no error - // so that it won't cause data sync - // failure. - PostProcessHashPartitionDataSyncTask( - std::move(data_sync_task), - data_sync_txm, - DataSyncTask::CkptErrorCode::NO_ERROR); - }, - 10000, - false); - - upload_batch_closure->SetChannel(node_id, channel); - - ins_res.first->second = upload_batch_closure; - req_ptr = upload_batch_closure->UploadBatchRequest(); - req_ptr->set_node_group_id(dest_ng); - req_ptr->set_node_group_term(-1); - req_ptr->set_partition_id(-1); - req_ptr->set_table_name_str(table_name.String()); - req_ptr->set_table_type( - remote::ToRemoteType::ConvertTableType( - table_name.Type())); - req_ptr->set_table_engine( - remote::ToRemoteType::ConvertTableEngine( - table_name.Engine())); - req_ptr->set_kind( - remote::UploadBatchKind::DIRTY_BUCKET_DATA); - req_ptr->set_batch_size(0); - // keys - req_ptr->clear_keys(); - // records - req_ptr->clear_records(); - // commit_ts - req_ptr->clear_commit_ts(); - // rec_status - req_ptr->clear_rec_status(); + req_ptr = ins_res.first->second->UploadBatchRequest(); + } + + if (req_ptr) + { + std::string *keys_str = req_ptr->mutable_keys(); + std::string *rec_status_str = + req_ptr->mutable_rec_status(); + std::string *commit_ts_str = + req_ptr->mutable_commit_ts(); + size_t len_sizeof = sizeof(uint64_t); + const char *val_ptr = nullptr; + ref.Key().Serialize(*keys_str); + if (ref.payload_status_ == RecordStatus::Normal) + { + std::string *recs_str = req_ptr->mutable_records(); + ref.Payload()->Serialize(*recs_str); + } + const char *status_ptr = reinterpret_cast( + &(ref.payload_status_)); + rec_status_str->append(status_ptr, + sizeof(RecordStatus)); + val_ptr = + reinterpret_cast(&(ref.commit_ts_)); + commit_ts_str->append(val_ptr, len_sizeof); + req_ptr->set_batch_size(req_ptr->batch_size() + 1); } } - else + meta_lk.unlock(); + { - req_ptr = ins_res.first->second->UploadBatchRequest(); + std::unique_lock flight_lk( + data_sync_task->flight_task_mux_); + data_sync_task->flight_task_cnt_ += + send_cache_closures.size(); } - - if (req_ptr) + // Send cache to target node groups. + for (auto &[ng, upload_batch_closure] : send_cache_closures) { - std::string *keys_str = req_ptr->mutable_keys(); - std::string *rec_status_str = req_ptr->mutable_rec_status(); - std::string *commit_ts_str = req_ptr->mutable_commit_ts(); - size_t len_sizeof = sizeof(uint64_t); - const char *val_ptr = nullptr; - ref.Key().Serialize(*keys_str); - if (ref.payload_status_ == RecordStatus::Normal) - { - std::string *recs_str = req_ptr->mutable_records(); - ref.Payload()->Serialize(*recs_str); - } - const char *status_ptr = - reinterpret_cast(&(ref.payload_status_)); - rec_status_str->append(status_ptr, sizeof(RecordStatus)); - val_ptr = reinterpret_cast(&(ref.commit_ts_)); - commit_ts_str->append(val_ptr, len_sizeof); - req_ptr->set_batch_size(req_ptr->batch_size() + 1); + remote::CcRpcService_Stub stub( + upload_batch_closure->Channel()); + brpc::Controller *cntl_ptr = + upload_batch_closure->Controller(); + cntl_ptr->set_timeout_ms(10000); + // Asynchronous mode + stub.UploadBatch( + upload_batch_closure->Controller(), + upload_batch_closure->UploadBatchRequest(), + upload_batch_closure->UploadBatchResponse(), + upload_batch_closure); } } - meta_lk.unlock(); + uint64_t flush_data_size = scan_cc.accumulated_flush_data_size_; + + // nothing to flush + if (scan_cc.accumulated_scan_cnt_ == 0) { - std::unique_lock flight_lk( - data_sync_task->flight_task_mux_); - data_sync_task->flight_task_cnt_ += send_cache_closures.size(); - } - // Send cache to target node groups. - for (auto &[ng, upload_batch_closure] : send_cache_closures) - { - remote::CcRpcService_Stub stub(upload_batch_closure->Channel()); - brpc::Controller *cntl_ptr = upload_batch_closure->Controller(); - cntl_ptr->set_timeout_ms(10000); - // Asynchronous mode - stub.UploadBatch(upload_batch_closure->Controller(), - upload_batch_closure->UploadBatchRequest(), - upload_batch_closure->UploadBatchResponse(), - upload_batch_closure); + scan_cc.Reset(); + continue; } - } - - uint64_t flush_data_size = scan_cc.accumulated_flush_data_size_; - // nothing to flush - if (scan_cc.accumulated_scan_cnt_ == 0) - { - scan_cc.Reset(); - continue; - } - - // The cost of FlushRecord also needs to be considered. + // The cost of FlushRecord also needs to be considered. #ifdef WITH_JEMALLOC - flush_data_size += (scan_cc.DataSyncVec().size() * sizeof(FlushRecord) + - scan_cc.ArchiveVec().size() * sizeof(FlushRecord) + - scan_cc.MoveBaseIdxVec().size() * - sizeof(std::pair)); + flush_data_size += + (scan_cc.DataSyncVec().size() * sizeof(FlushRecord) + + scan_cc.ArchiveVec().size() * sizeof(FlushRecord) + + scan_cc.MoveBaseIdxVec().size() * + sizeof(std::pair)); #else - // Check if vectors are empty before calling malloc_usable_size - // to avoid SEGV on nullptr or invalid pointers. - // Use malloc_usable_size when ASan is enabled (vectors may be - // allocated by ASan's allocator), otherwise use - // mi_malloc_usable_size for mimalloc-allocated memory. - { - auto &data_sync_vec_ref = scan_cc.DataSyncVec(); - auto &archive_vec_ref = scan_cc.ArchiveVec(); - auto &move_base_idx_vec_ref = scan_cc.MoveBaseIdxVec(); + // Check if vectors are empty before calling malloc_usable_size + // to avoid SEGV on nullptr or invalid pointers. + // Use malloc_usable_size when ASan is enabled (vectors may be + // allocated by ASan's allocator), otherwise use + // mi_malloc_usable_size for mimalloc-allocated memory. + { + auto &data_sync_vec_ref = scan_cc.DataSyncVec(); + auto &archive_vec_ref = scan_cc.ArchiveVec(); + auto &move_base_idx_vec_ref = scan_cc.MoveBaseIdxVec(); #ifdef __SANITIZE_ADDRESS__ - // When ASan is enabled, use standard malloc_usable_size - flush_data_size += - (data_sync_vec_ref.empty() - ? 0 - : malloc_usable_size(data_sync_vec_ref.data())) + - (archive_vec_ref.empty() - ? 0 - : malloc_usable_size(archive_vec_ref.data())) + - (move_base_idx_vec_ref.empty() - ? 0 - : malloc_usable_size(move_base_idx_vec_ref.data())); + // When ASan is enabled, use standard malloc_usable_size + flush_data_size += + (data_sync_vec_ref.empty() + ? 0 + : malloc_usable_size(data_sync_vec_ref.data())) + + (archive_vec_ref.empty() + ? 0 + : malloc_usable_size(archive_vec_ref.data())) + + (move_base_idx_vec_ref.empty() + ? 0 + : malloc_usable_size(move_base_idx_vec_ref.data())); #else - // When ASan is not enabled, use mimalloc's API - flush_data_size += - (data_sync_vec_ref.empty() - ? 0 - : mi_malloc_usable_size(data_sync_vec_ref.data())) + - (archive_vec_ref.empty() - ? 0 - : mi_malloc_usable_size(archive_vec_ref.data())) + - (move_base_idx_vec_ref.empty() - ? 0 - : mi_malloc_usable_size(move_base_idx_vec_ref.data())); + // When ASan is not enabled, use mimalloc's API + flush_data_size += + (data_sync_vec_ref.empty() + ? 0 + : mi_malloc_usable_size(data_sync_vec_ref.data())) + + (archive_vec_ref.empty() + ? 0 + : mi_malloc_usable_size(archive_vec_ref.data())) + + (move_base_idx_vec_ref.empty() + ? 0 + : mi_malloc_usable_size(move_base_idx_vec_ref.data())); #endif - } + } #endif - // this thread will wait in AllocatePendingFlushDataMemQuota if - // quota is not available - uint64_t old_usage = - data_sync_mem_controller_.AllocateFlushDataMemQuota( - flush_data_size); - DLOG(INFO) << "AllocateFlushDataMemQuota old_usage: " << old_usage - << " new_usage: " << old_usage + flush_data_size - << " quota: " << data_sync_mem_controller_.FlushMemoryQuota() - << " flight_tasks: " << data_sync_task->flight_task_cnt_ - << " record count: " << scan_cc.accumulated_scan_cnt_; - - std::unique_lock heap_lk(hash_partition_ckpt_heap_mux_); - mi_threadid_t prev_thd = - mi_override_thread(hash_partition_main_thread_id_); - mi_heap_t *prev_heap = mi_heap_set_default(hash_partition_ckpt_heap_); + uint64_t old_usage = + data_sync_mem_controller_.AllocateFlushDataMemQuota( + flush_data_size); + DLOG(INFO) << "AllocateFlushDataMemQuota old_usage: " << old_usage + << " new_usage: " << old_usage + flush_data_size + << " quota: " + << data_sync_mem_controller_.FlushMemoryQuota() + << " flight_tasks: " << data_sync_task->flight_task_cnt_ + << " record count: " << scan_cc.accumulated_scan_cnt_; + + std::unique_lock heap_lk(hash_partition_ckpt_heap_mux_); + mi_threadid_t prev_thd = + mi_override_thread(hash_partition_main_thread_id_); + mi_heap_t *prev_heap = + mi_heap_set_default(hash_partition_ckpt_heap_); #if defined(WITH_JEMALLOC) - uint32_t prev_arena; - JemallocArenaSwitcher::ReadCurrentArena(prev_arena); - JemallocArenaSwitcher::SwitchToArena(hash_partition_ckpt_arena_id_); + uint32_t prev_arena; + JemallocArenaSwitcher::ReadCurrentArena(prev_arena); + JemallocArenaSwitcher::SwitchToArena(hash_partition_ckpt_arena_id_); #endif - data_sync_vec->reserve(scan_cc.accumulated_scan_cnt_); - for (size_t j = 0; j < scan_cc.accumulated_scan_cnt_; ++j) - { - auto &rec = scan_cc.DataSyncVec()[j]; - // Note. Clone key instead of move key. The memory of - // rec.Key() will be reused to avoid memory allocation. - if (rec.cce_) + data_sync_vec->reserve(scan_cc.accumulated_scan_cnt_); + for (size_t j = 0; j < scan_cc.accumulated_scan_cnt_; ++j) { - // cce_ is null means the key is already persisted on - // kv, so we don't need to put it into the flush vec. - int32_t part_id = - Sharder::MapKeyHashToHashPartitionId(rec.Key().Hash()); - if (table_name.Engine() == TableEngine::EloqKv) - { - data_sync_vec->emplace_back(rec.Key().Clone(), - rec.GetNonVersionedPayload(), - rec.payload_status_, - rec.commit_ts_, - rec.cce_, - rec.post_flush_size_, - part_id); - } - else + auto &rec = scan_cc.DataSyncVec()[j]; + // Note. Clone key instead of move key. The memory of + // rec.Key() will be reused to avoid memory allocation. + if (rec.cce_) { - data_sync_vec->emplace_back(rec.Key().Clone(), - rec.ReleaseVersionedPayload(), - rec.payload_status_, - rec.commit_ts_, - rec.cce_, - rec.post_flush_size_, - part_id); + // cce_ is null means the key is already persisted on + // kv, so we don't need to put it into the flush vec. + int32_t part_id = + Sharder::MapKeyHashToHashPartitionId(rec.Key().Hash()); + if (table_name.Engine() == TableEngine::EloqKv) + { + data_sync_vec->emplace_back( + rec.Key().Clone(), + rec.GetNonVersionedPayload(), + rec.payload_status_, + rec.commit_ts_, + rec.cce_, + rec.post_flush_size_, + part_id); + } + else + { + data_sync_vec->emplace_back( + rec.Key().Clone(), + rec.ReleaseVersionedPayload(), + rec.payload_status_, + rec.commit_ts_, + rec.cce_, + rec.post_flush_size_, + part_id); + } } } - } - for (size_t j = 0; j < scan_cc.ArchiveVec().size(); ++j) - { - auto &rec = scan_cc.ArchiveVec()[j]; - // Note. We need to ensure the copy constructor of - // FlushRecord could not be called. - rec.SetKey((*data_sync_vec)[rec.GetKeyIndex()].Key()); - } + for (size_t j = 0; j < scan_cc.ArchiveVec().size(); ++j) + { + auto &rec = scan_cc.ArchiveVec()[j]; + // Note. We need to ensure the copy constructor of + // FlushRecord could not be called. + rec.SetKey((*data_sync_vec)[rec.GetKeyIndex()].Key()); + } - for (size_t j = 0; j < scan_cc.MoveBaseIdxVec().size(); ++j) - { - size_t key_idx = scan_cc.MoveBaseIdxVec()[j]; - TxKey key_raw = (*data_sync_vec)[key_idx].Key(); - int32_t part_id = - Sharder::MapKeyHashToHashPartitionId(key_raw.Hash()); - mv_base_vec->emplace_back(std::move(key_raw), part_id); - } - mi_override_thread(prev_thd); - mi_heap_set_default(prev_heap); + for (size_t j = 0; j < scan_cc.MoveBaseIdxVec().size(); ++j) + { + size_t key_idx = scan_cc.MoveBaseIdxVec()[j]; + TxKey key_raw = (*data_sync_vec)[key_idx].Key(); + int32_t part_id = + Sharder::MapKeyHashToHashPartitionId(key_raw.Hash()); + mv_base_vec->emplace_back(std::move(key_raw), part_id); + } + mi_override_thread(prev_thd); + mi_heap_set_default(prev_heap); #if defined(WITH_JEMALLOC) - // override arena id - JemallocArenaSwitcher::SwitchToArena(prev_arena); + // override arena id + JemallocArenaSwitcher::SwitchToArena(prev_arena); #endif - heap_lk.unlock(); + heap_lk.unlock(); - std::move(scan_cc.ArchiveVec().begin(), - scan_cc.ArchiveVec().end(), - std::back_inserter(*archive_vec)); + std::move(scan_cc.ArchiveVec().begin(), + scan_cc.ArchiveVec().end(), + std::back_inserter(*archive_vec)); - vec_mem_usage += flush_data_size; + vec_mem_usage += flush_data_size; - scan_data_drained = scan_cc.IsDrained() && scan_data_drained; + scan_data_drained = scan_cc.IsDrained() && scan_data_drained; - { - std::unique_lock flight_task_lk( - data_sync_task->flight_task_mux_); - if (data_sync_task->ckpt_err_ == - DataSyncTask::CkptErrorCode::FLUSH_ERROR) { - flight_task_lk.unlock(); - - LOG(WARNING) - << "There are error during flush for this data sync: " - << data_sync_txm->TxNumber() << " on worker#" << worker_idx - << ". Terminal this datasync task."; - // 1. Release read intent on paused key - if (!scan_cc.IsDrained()) + std::unique_lock flight_task_lk( + data_sync_task->flight_task_mux_); + if (data_sync_task->ckpt_err_ == + DataSyncTask::CkptErrorCode::FLUSH_ERROR) { - scan_cc.Reset( - HashPartitionDataSyncScanCc::OpType::Terminated); - EnqueueLowPriorityCcRequestToShard(worker_idx, &scan_cc); - scan_cc.Wait(); + flight_task_lk.unlock(); + + LOG(WARNING) + << "There are error during flush for this data sync: " + << data_sync_txm->TxNumber() << " on worker#" + << worker_idx << ". Terminal this datasync task."; + // 1. Release read intent on paused key + if (!scan_cc.IsDrained()) + { + scan_cc.Reset( + HashPartitionDataSyncScanCc::OpType::Terminated); + EnqueueLowPriorityCcRequestToShard(worker_idx, + &scan_cc); + scan_cc.Wait(); + } + // 2. Release memory usage on this datasync worker. + data_sync_vec->clear(); + archive_vec->clear(); + mv_base_vec->clear(); + data_sync_mem_controller_.DeallocateFlushMemQuota( + vec_mem_usage); + break; } - // 2. Release memory usage on this datasync worker. - data_sync_vec->clear(); - archive_vec->clear(); - mv_base_vec->clear(); - data_sync_mem_controller_.DeallocateFlushMemQuota( - vec_mem_usage); - break; - } - - // Flush worker will call PostProcessDataSyncTask() to - // decrement flight task count. - data_sync_task->flight_task_cnt_ += 1; - } - - AddFlushTaskEntry( - std::make_unique(std::move(data_sync_vec), - std::move(archive_vec), - std::move(mv_base_vec), - data_sync_txm, - data_sync_task, - catalog_rec.CopySchema(), - vec_mem_usage)); - - data_sync_vec = std::make_unique>(); - - archive_vec = std::make_unique>(); - - mv_base_vec = - std::make_unique>>(); - - vec_mem_usage = 0; - - if (scan_cc.scan_heap_is_full_ == 1) - { - // Clear the FlushRecords' memory of scan cc since the - // DataSyncScan heap is full. - auto &data_sync_vec_ref = scan_cc.DataSyncVec(); - auto &archive_vec_ref = scan_cc.ArchiveVec(); - ReleaseDataSyncScanHeapCc release_scan_heap_cc(&data_sync_vec_ref, - &archive_vec_ref); - EnqueueLowPriorityCcRequestToShard(worker_idx, - &release_scan_heap_cc); - release_scan_heap_cc.Wait(); - } - - scan_cc.Reset(); - } - - // release scan heap memory after scan finish - auto &data_sync_vec_ref = scan_cc.DataSyncVec(); - auto &archive_vec_ref = scan_cc.ArchiveVec(); - ReleaseDataSyncScanHeapCc release_scan_heap_cc(&data_sync_vec_ref, - &archive_vec_ref); - EnqueueLowPriorityCcRequestToShard(worker_idx, &release_scan_heap_cc); - release_scan_heap_cc.Wait(); - if (!data_sync_vec->empty() || !archive_vec->empty() || - !mv_base_vec->empty()) - { - std::unique_lock flight_task_lk( - data_sync_task->flight_task_mux_); - if (data_sync_task->ckpt_err_ == DataSyncTask::CkptErrorCode::NO_ERROR) - { - data_sync_task->flight_task_cnt_ += 1; - flight_task_lk.unlock(); + // Flush worker will call PostProcessDataSyncTask() to + // decrement flight task count. + data_sync_task->flight_task_cnt_ += 1; + } AddFlushTaskEntry( std::make_unique(std::move(data_sync_vec), @@ -5179,13 +5220,67 @@ void LocalCcShards::DataSyncForHashPartition( data_sync_task, catalog_rec.CopySchema(), vec_mem_usage)); + + data_sync_vec = std::make_unique>(); + + archive_vec = std::make_unique>(); + + mv_base_vec = + std::make_unique>>(); + + vec_mem_usage = 0; + + if (scan_cc.scan_heap_is_full_ == 1) + { + // Clear the FlushRecords' memory of scan cc since the + // DataSyncScan heap is full. + auto &data_sync_vec_ref = scan_cc.DataSyncVec(); + auto &archive_vec_ref = scan_cc.ArchiveVec(); + ReleaseDataSyncScanHeapCc release_scan_heap_cc( + &data_sync_vec_ref, &archive_vec_ref); + EnqueueLowPriorityCcRequestToShard(worker_idx, + &release_scan_heap_cc); + release_scan_heap_cc.Wait(); + } + + scan_cc.Reset(); } - else + + // release scan heap memory after scan finish + auto &data_sync_vec_ref = scan_cc.DataSyncVec(); + auto &archive_vec_ref = scan_cc.ArchiveVec(); + ReleaseDataSyncScanHeapCc release_scan_heap_cc(&data_sync_vec_ref, + &archive_vec_ref); + EnqueueLowPriorityCcRequestToShard(worker_idx, &release_scan_heap_cc); + release_scan_heap_cc.Wait(); + + if (!data_sync_vec->empty() || !archive_vec->empty() || + !mv_base_vec->empty()) { - // There are error during flush, and if we do not put the - // current batch data into flush worker, should release the - // memory usage. - data_sync_mem_controller_.DeallocateFlushMemQuota(vec_mem_usage); + std::unique_lock flight_task_lk( + data_sync_task->flight_task_mux_); + if (data_sync_task->ckpt_err_ == + DataSyncTask::CkptErrorCode::NO_ERROR) + { + data_sync_task->flight_task_cnt_ += 1; + flight_task_lk.unlock(); + AddFlushTaskEntry( + std::make_unique(std::move(data_sync_vec), + std::move(archive_vec), + std::move(mv_base_vec), + data_sync_txm, + data_sync_task, + catalog_rec.CopySchema(), + vec_mem_usage)); + } + else + { + // There are error during flush, and if we do not put the + // current batch data into flush worker, should release the + // memory usage. + data_sync_mem_controller_.DeallocateFlushMemQuota( + vec_mem_usage); + } } } @@ -5663,80 +5758,106 @@ void LocalCcShards::SplitFlushRange( txservice::CommitTx(split_txm); } +size_t LocalCcShards::DataSyncWorkerToFlushDataWorker( + size_t data_sync_worker_id) const +{ + return data_sync_worker_id % flush_data_worker_ctx_.worker_num_; +} + void LocalCcShards::AddFlushTaskEntry(std::unique_ptr &&entry) { - cur_flush_buffer_.AddFlushTaskEntry(std::move(entry)); + assert(cur_flush_buffers_.size() == + static_cast(flush_data_worker_ctx_.worker_num_)); + assert(pending_flush_work_.size() == + static_cast(flush_data_worker_ctx_.worker_num_)); + assert(entry->data_sync_task_ != nullptr); + + // Compute target buffer/queue: map data_sync task id to fixed flush worker + const auto &data_sync_task = entry->data_sync_task_; + size_t target = DataSyncWorkerToFlushDataWorker(static_cast( + data_sync_task->id_ % data_sync_worker_ctx_.worker_num_)); + + std::unique_lock worker_lk(flush_data_worker_ctx_.mux_); + auto &cur_flush_buffer = *cur_flush_buffers_[target]; + + cur_flush_buffer.AddFlushTaskEntry(std::move(entry)); std::unique_ptr flush_data_task = - cur_flush_buffer_.MoveFlushData(false); + cur_flush_buffer.MoveFlushData(false); if (flush_data_task != nullptr) { - std::unique_lock worker_lk(flush_data_worker_ctx_.mux_); + auto &pending_flush_work = pending_flush_work_[target]; // Try to merge with the last task if queue is not empty - if (!pending_flush_work_.empty()) + if (!pending_flush_work.empty()) { - auto &last_task = pending_flush_work_.back(); + auto &last_task = pending_flush_work.back(); if (last_task->MergeFrom(std::move(flush_data_task))) { // Merge successful, task was merged into last_task - flush_data_worker_ctx_.cv_.notify_one(); + flush_data_worker_ctx_.cv_.notify_all(); return; } } // Could not merge, wait if queue is full - while (pending_flush_work_.size() >= - static_cast(flush_data_worker_ctx_.worker_num_)) + while (pending_flush_work.size() >= 2) { flush_data_worker_ctx_.cv_.wait(worker_lk); } // Add as new task - pending_flush_work_.emplace_back(std::move(flush_data_task)); - flush_data_worker_ctx_.cv_.notify_one(); + pending_flush_work.emplace_back(std::move(flush_data_task)); + flush_data_worker_ctx_.cv_.notify_all(); } } void LocalCcShards::FlushCurrentFlushBuffer() { - std::unique_ptr flush_data_task = - cur_flush_buffer_.MoveFlushData(true); - if (flush_data_task != nullptr) + assert(cur_flush_buffers_.size() == flush_data_worker_ctx_.worker_num_); + assert(pending_flush_work_.size() == flush_data_worker_ctx_.worker_num_); + + std::unique_lock worker_lk(flush_data_worker_ctx_.mux_); + // Flush all workers' buffers + for (int worker_idx = 0; worker_idx < flush_data_worker_ctx_.worker_num_; + ++worker_idx) { - std::unique_lock worker_lk(flush_data_worker_ctx_.mux_); + auto &cur_flush_buffer = *cur_flush_buffers_[worker_idx]; + auto &pending_flush_work = pending_flush_work_[worker_idx]; - // Try to merge with the last task if queue is not empty - if (!pending_flush_work_.empty()) + std::unique_ptr flush_data_task = + cur_flush_buffer.MoveFlushData(true); + if (flush_data_task != nullptr) { - auto &last_task = pending_flush_work_.back(); - if (last_task->MergeFrom(std::move(flush_data_task))) + // Try to merge with the last task if queue is not empty + if (!pending_flush_work.empty()) { - // Merge successful, task was merged into last_task - flush_data_worker_ctx_.cv_.notify_one(); - return; + auto &last_task = pending_flush_work.back(); + if (last_task->MergeFrom(std::move(flush_data_task))) + { + // Merge successful, task was merged into last_task + flush_data_worker_ctx_.cv_.notify_all(); + continue; + } } - } - // Could not merge, wait if queue is full - while (pending_flush_work_.size() >= - static_cast(flush_data_worker_ctx_.worker_num_)) - { - flush_data_worker_ctx_.cv_.wait(worker_lk); + // Add as new task + pending_flush_work.emplace_back(std::move(flush_data_task)); + flush_data_worker_ctx_.cv_.notify_all(); } - - // Add as new task - pending_flush_work_.emplace_back(std::move(flush_data_task)); - flush_data_worker_ctx_.cv_.notify_one(); } } -void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk) +void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk, + size_t worker_idx) { + assert(worker_idx < pending_flush_work_.size()); + auto &pending_flush_work = pending_flush_work_[worker_idx]; + // Retrieve first pending work and pop it (FIFO). std::unique_ptr cur_work = - std::move(pending_flush_work_.front()); - pending_flush_work_.pop_front(); + std::move(pending_flush_work.front()); + pending_flush_work.pop_front(); // Notify any threads waiting for queue space flush_data_worker_ctx_.cv_.notify_all(); @@ -5746,7 +5867,7 @@ void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk) auto &flush_task_entries = cur_work->flush_task_entries_; bool succ = true; - // Flushes to the data store + if (EnableMvcc()) { succ = store_hd_->CopyBaseToArchive(flush_task_entries); @@ -5777,7 +5898,6 @@ void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk) } } - // Persist data in kv store if needed if (succ && store_hd_->NeedPersistKV()) { std::vector kv_table_names; @@ -5905,19 +6025,33 @@ void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk) flush_worker_lk.lock(); } -void LocalCcShards::FlushDataWorker() +void LocalCcShards::FlushDataWorker(size_t worker_idx) { + assert(worker_idx < + static_cast(flush_data_worker_ctx_.worker_num_)); + assert(worker_idx < static_cast(pending_flush_work_.size())); + assert(worker_idx < static_cast(cur_flush_buffers_.size())); + + auto &pending_flush_work = pending_flush_work_[worker_idx]; + auto &cur_flush_buffer = *cur_flush_buffers_[worker_idx]; + + using clock = std::chrono::steady_clock; size_t previous_flush_size = 0; - auto previous_size_update_time = std::chrono::steady_clock::now(); + auto previous_size_update_time = clock::now(); + std::unique_lock flush_worker_lk(flush_data_worker_ctx_.mux_); while (flush_data_worker_ctx_.status_ == WorkerStatus::Active) { flush_data_worker_ctx_.cv_.wait_for( flush_worker_lk, 10s, - [this, &previous_flush_size, &previous_size_update_time] + [this, + &pending_flush_work, + &cur_flush_buffer, + &previous_flush_size, + &previous_size_update_time] { - if (!pending_flush_work_.empty() || + if (!pending_flush_work.empty() || flush_data_worker_ctx_.status_ == WorkerStatus::Terminated) { return true; @@ -5926,7 +6060,7 @@ void LocalCcShards::FlushDataWorker() if (current_time - previous_size_update_time > 10s) { size_t current_flush_size = - cur_flush_buffer_.GetPendingFlushSize(); + cur_flush_buffer.GetPendingFlushSize(); bool flush_size_changed = current_flush_size != previous_flush_size; previous_flush_size = current_flush_size; @@ -5938,15 +6072,15 @@ void LocalCcShards::FlushDataWorker() // read lock held by ongoing data sync tx, which might // block the DDL. std::unique_ptr flush_data_task = - cur_flush_buffer_.MoveFlushData(true); + cur_flush_buffer.MoveFlushData(true); if (flush_data_task != nullptr) { // Try to merge with the last task if queue is not // empty Note: flush_worker_lk is already held here // (inside condition variable predicate) - if (!pending_flush_work_.empty()) + if (!pending_flush_work.empty()) { - auto &last_task = pending_flush_work_.back(); + auto &last_task = pending_flush_work.back(); if (last_task->MergeFrom( std::move(flush_data_task))) { @@ -5957,8 +6091,8 @@ void LocalCcShards::FlushDataWorker() } // Add as new task. We just checked that - // pending_flush_work_ is empty, - pending_flush_work_.emplace_back( + // pending_flush_work is empty, + pending_flush_work.emplace_back( std::move(flush_data_task)); return true; } @@ -5968,17 +6102,17 @@ void LocalCcShards::FlushDataWorker() return false; }); - if (pending_flush_work_.empty()) + if (pending_flush_work.empty()) { continue; } - FlushData(flush_worker_lk); + FlushData(flush_worker_lk, worker_idx); } - while (!pending_flush_work_.empty()) + while (!pending_flush_work.empty()) { - FlushData(flush_worker_lk); + FlushData(flush_worker_lk, worker_idx); } } From edfb7611d9f81aec948c3b2ecb6ba0a268f6f684 Mon Sep 17 00:00:00 2001 From: lokax Date: Thu, 26 Feb 2026 14:00:05 +0800 Subject: [PATCH 02/38] phase1-2 --- build_tx_service.cmake | 15 +++++- tx_service/include/cc/local_cc_shards.h | 8 +++ tx_service/src/cc/local_cc_shards.cpp | 72 ++++++++++++++++++++----- 3 files changed, 79 insertions(+), 16 deletions(-) diff --git a/build_tx_service.cmake b/build_tx_service.cmake index 1857afa4..f0ae597f 100644 --- a/build_tx_service.cmake +++ b/build_tx_service.cmake @@ -30,6 +30,17 @@ find_package(Protobuf REQUIRED) find_package(GFLAGS REQUIRED) find_package(MIMALLOC REQUIRED) +# boost_context for FlushDataWorker coroutine refactor (Phase 1) +if(CMAKE_BUILD_TYPE STREQUAL "Debug" AND CMAKE_CXX_FLAGS MATCHES "fsanitize=address") + find_library(Boost_CONTEXT_LIBRARY NAMES boost_context-asan) +else() + find_library(Boost_CONTEXT_LIBRARY NAMES boost_context) +endif() +if(NOT Boost_CONTEXT_LIBRARY) + message(FATAL_ERROR "libboost_context not found") +endif() +find_package(Boost 1.70 REQUIRED) + find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h) find_library(GFLAGS_LIBRARY NAMES gflags libgflags) @@ -211,9 +222,9 @@ set(ELOQ_SOURCES ${ELOQ_SOURCES} ${PROTO_CC_FILES}) ADD_LIBRARY(txservice ${ELOQ_SOURCES}) -target_include_directories(txservice PUBLIC ${INCLUDE_DIR}) +target_include_directories(txservice PUBLIC ${INCLUDE_DIR} ${Boost_INCLUDE_DIRS}) -target_link_libraries(txservice PUBLIC ${LINK_LIB} ${PROTOBUF_LIBRARIES}) +target_link_libraries(txservice PUBLIC ${LINK_LIB} ${PROTOBUF_LIBRARIES} ${Boost_CONTEXT_LIBRARY}) if(WITH_JEMALLOC) target_link_libraries(txservice PUBLIC jemalloc_cfg) diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index 0fc899da..043a282f 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -2510,6 +2510,14 @@ class LocalCcShards void FlushData(std::unique_lock &flush_worker_lk, size_t worker_idx); + bool ShouldYieldFlushData(size_t worker_idx) const; + void FlushDataImpl( + FlushDataTask *cur_work, + size_t worker_idx, + const std::function &sync_yield_func, + const std::function &yield_fn, + const std::function &resume_fn); + // Memory controller for data sync. DataSyncMemoryController data_sync_mem_controller_; diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index df856953..620d3a13 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -5848,26 +5848,28 @@ void LocalCcShards::FlushCurrentFlushBuffer() } } -void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk, - size_t worker_idx) +bool LocalCcShards::ShouldYieldFlushData(size_t worker_idx) const { - assert(worker_idx < pending_flush_work_.size()); - auto &pending_flush_work = pending_flush_work_[worker_idx]; - - // Retrieve first pending work and pop it (FIFO). - std::unique_ptr cur_work = - std::move(pending_flush_work.front()); - pending_flush_work.pop_front(); - - // Notify any threads waiting for queue space - flush_data_worker_ctx_.cv_.notify_all(); + return !pending_flush_work_[worker_idx].empty(); +} - flush_worker_lk.unlock(); +void LocalCcShards::FlushDataImpl( + FlushDataTask *cur_work, + size_t worker_idx, + const std::function &sync_yield_func, + const std::function &yield_fn, + const std::function &resume_fn) +{ + (void)yield_fn; + (void)resume_fn; auto &flush_task_entries = cur_work->flush_task_entries_; - bool succ = true; + if (ShouldYieldFlushData(worker_idx)) + { + sync_yield_func(); + } if (EnableMvcc()) { succ = store_hd_->CopyBaseToArchive(flush_task_entries); @@ -5878,6 +5880,10 @@ void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk, } } + if (ShouldYieldFlushData(worker_idx)) + { + sync_yield_func(); + } if (succ) { succ = store_hd_->PutAll(flush_task_entries); @@ -5888,6 +5894,10 @@ void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk, } } + if (ShouldYieldFlushData(worker_idx)) + { + sync_yield_func(); + } if (succ && EnableMvcc()) { succ = store_hd_->PutArchivesAll(flush_task_entries); @@ -5898,6 +5908,10 @@ void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk, } } + if (ShouldYieldFlushData(worker_idx)) + { + sync_yield_func(); + } if (succ && store_hd_->NeedPersistKV()) { std::vector kv_table_names; @@ -5939,6 +5953,10 @@ void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk, { for (auto &[kv_table_name, entries] : flush_task_entries) { + if (ShouldYieldFlushData(worker_idx)) + { + sync_yield_func(); + } for (auto &entry : entries) { if (!entry->data_sync_task_->need_update_ckpt_ts_) @@ -5995,6 +6013,10 @@ void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk, } } + if (ShouldYieldFlushData(worker_idx)) + { + sync_yield_func(); + } // Notify cc shards that dirty data has been flushed. This will re-enqueue // kickout data cc reqs if there are any. WaitableCc reset_cc( @@ -6022,6 +6044,28 @@ void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk, << " quota: " << data_sync_mem_controller_.FlushMemoryQuota(); PostProcessFlushTaskEntries(flush_task_entries, ckpt_err); +} + +void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk, + size_t worker_idx) +{ + assert(worker_idx < pending_flush_work_.size()); + auto &pending_flush_work = pending_flush_work_[worker_idx]; + + // Retrieve first pending work and pop it (FIFO). + std::unique_ptr cur_work = + std::move(pending_flush_work.front()); + pending_flush_work.pop_front(); + + // Notify any threads waiting for queue space + flush_data_worker_ctx_.cv_.notify_all(); + + flush_worker_lk.unlock(); + + // Phase 2: Pass no-op callbacks; Phase 4/5 will pass real coroutine + // callbacks. + FlushDataImpl(cur_work.get(), worker_idx, []() {}, []() {}, []() {}); + flush_worker_lk.lock(); } From 9bdfca4a33e1c51429eb786153c77b017573fa85 Mon Sep 17 00:00:00 2001 From: lokax Date: Thu, 26 Feb 2026 14:28:25 +0800 Subject: [PATCH 03/38] phase3 --- tx_service/include/cc/local_cc_shards.h | 14 ++++++++++++++ tx_service/src/cc/local_cc_shards.cpp | 1 + 2 files changed, 15 insertions(+) diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index 043a282f..e41683a2 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -63,6 +63,8 @@ #include "tx_start_ts_collector.h" #include "type.h" +#include + #ifdef WITH_JEMALLOC // we use uint32_t instead of unsigned to make the code more readable static_assert(std::is_same_v, @@ -85,6 +87,15 @@ class SkGenerator; class UploadBatchSlicesClosure; struct FlushDataTask; +struct CoroCtx +{ + boost::context::continuation coro_; + std::unique_ptr task_; + std::function sync_yield_func; // 同步 yield：入队后让出 + std::function yield_fn; // 异步 yield + std::function resume_fn; // 用 weak_ptr 捕获 ctx，无参调用 +}; + struct DataMigrationStatus { public: @@ -2505,6 +2516,9 @@ class LocalCcShards // Per-worker flush task queues. Each FlushDataWorker processes its // corresponding queue. std::vector>> pending_flush_work_; + // Per-worker queues of coroutine contexts ready to resume (yielded by + // sync_yield_func or resume_fn). + std::vector>> resume_queue_; void FlushDataWorker(size_t worker_idx); void FlushData(std::unique_lock &flush_worker_lk, diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 620d3a13..e8e93534 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -305,6 +305,7 @@ void LocalCcShards::StartBackgroudWorkers() std::make_unique(buffer_size)); } pending_flush_work_.resize(worker_num); + resume_queue_.resize(worker_num); // Starts flush worker threads firstly. for (int id = 0; id < flush_data_worker_ctx_.worker_num_; id++) From dbe231652bb5871208d60bd5ee45f5d5af337ab4 Mon Sep 17 00:00:00 2001 From: lokax Date: Thu, 26 Feb 2026 14:47:02 +0800 Subject: [PATCH 04/38] phase4-5 --- tx_service/include/cc/local_cc_shards.h | 10 +++ tx_service/src/cc/local_cc_shards.cpp | 86 ++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index e41683a2..da8fb7e8 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -64,6 +64,8 @@ #include "type.h" #include +#include +#include #ifdef WITH_JEMALLOC // we use uint32_t instead of unsigned to make the code more readable @@ -2520,6 +2522,14 @@ class LocalCcShards // sync_yield_func or resume_fn). std::vector>> resume_queue_; +#ifndef NDEBUG + boost::context::protected_fixedsize_stack flush_coro_stack_allocator_{ + 256 * 1024}; // 256KB, guard page for stack overflow detection +#else + boost::context::pooled_fixedsize_stack flush_coro_stack_allocator_{ + 256 * 1024}; // 256KB for FlushData call chain +#endif + void FlushDataWorker(size_t worker_idx); void FlushData(std::unique_lock &flush_worker_lk, size_t worker_idx); diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index e8e93534..157c3e66 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -6076,28 +6076,48 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) static_cast(flush_data_worker_ctx_.worker_num_)); assert(worker_idx < static_cast(pending_flush_work_.size())); assert(worker_idx < static_cast(cur_flush_buffers_.size())); + assert(worker_idx < static_cast(resume_queue_.size())); auto &pending_flush_work = pending_flush_work_[worker_idx]; auto &cur_flush_buffer = *cur_flush_buffers_[worker_idx]; + auto &resume_queue = resume_queue_[worker_idx]; using clock = std::chrono::steady_clock; + using continuation = boost::context::continuation; size_t previous_flush_size = 0; auto previous_size_update_time = clock::now(); std::unique_lock flush_worker_lk(flush_data_worker_ctx_.mux_); while (flush_data_worker_ctx_.status_ == WorkerStatus::Active) { + // 1. 从 resume_queue 获取被唤醒的协程 + if (!resume_queue.empty()) + { + std::shared_ptr ctx = + std::move(resume_queue.front()); + resume_queue.pop_front(); + flush_worker_lk.unlock(); + + ctx->coro_ = ctx->coro_.resume(); // 单次 resume，无循环 + + flush_worker_lk.lock(); + continue; + } + + // 2. 等待新任务或 resume_queue 通知 flush_data_worker_ctx_.cv_.wait_for( flush_worker_lk, 10s, [this, &pending_flush_work, + &resume_queue, &cur_flush_buffer, &previous_flush_size, &previous_size_update_time] { - if (!pending_flush_work.empty() || - flush_data_worker_ctx_.status_ == WorkerStatus::Terminated) + if (!pending_flush_work.empty() || !resume_queue.empty() || + flush_data_worker_ctx_.status_ == + WorkerStatus::Terminated) { return true; } @@ -6152,9 +6172,69 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) continue; } - FlushData(flush_worker_lk, worker_idx); + // 3. 取任务，以协程执行 FlushData + std::unique_ptr cur_work = + std::move(pending_flush_work.front()); + pending_flush_work.pop_front(); + flush_data_worker_ctx_.cv_.notify_all(); + flush_worker_lk.unlock(); + + auto ctx = std::make_shared(); + ctx->task_ = std::move(cur_work); + ctx->coro_ = boost::context::callcc( + std::allocator_arg, + flush_coro_stack_allocator_, + [this, ctx, worker_idx](continuation &&sink) + { + ctx->yield_fn = [&sink]() { sink = sink.resume(); }; + ctx->sync_yield_func = + [&sink, weak_ctx = std::weak_ptr(ctx), this, + worker_idx]() + { + if (auto c = weak_ctx.lock()) + { + { + std::lock_guard lk( + flush_data_worker_ctx_.mux_); + resume_queue_[worker_idx].push_back(std::move(c)); + flush_data_worker_ctx_.cv_.notify_all(); + } + sink = sink.resume(); + } + }; + ctx->resume_fn = + [this, worker_idx, + weak_ctx = std::weak_ptr(ctx)]() + { + if (auto c = weak_ctx.lock()) + { + std::lock_guard lk( + flush_data_worker_ctx_.mux_); + resume_queue_[worker_idx].push_back(std::move(c)); + flush_data_worker_ctx_.cv_.notify_all(); + } + }; + FlushDataImpl(ctx->task_.get(), + worker_idx, + ctx->sync_yield_func, + ctx->yield_fn, + ctx->resume_fn); + return std::move(sink); + }); + + flush_worker_lk.lock(); } + // Terminate 时清空队列 + while (!resume_queue.empty()) + { + std::shared_ptr ctx = + std::move(resume_queue.front()); + resume_queue.pop_front(); + flush_worker_lk.unlock(); + ctx->coro_ = ctx->coro_.resume(); + flush_worker_lk.lock(); + } while (!pending_flush_work.empty()) { FlushData(flush_worker_lk, worker_idx); From 93784b8cee50c0924dbb4811a78b7e1ab2166bf7 Mon Sep 17 00:00:00 2001 From: lokax Date: Thu, 26 Feb 2026 15:59:29 +0800 Subject: [PATCH 05/38] putall phase1 --- store_handler/data_store_service_client.cpp | 28 ++++++++++--- store_handler/data_store_service_client.h | 7 ++++ .../data_store_service_client_closure.cpp | 28 +++++++++++++ .../data_store_service_client_closure.h | 41 ++++++++++++++++++- tx_service/include/cc/local_cc_shards.h | 24 +++++------ 5 files changed, 108 insertions(+), 20 deletions(-) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index 10052f23..7a2fddd0 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -314,6 +314,16 @@ bool DataStoreServiceClient::PutAll( std::unordered_map>> &flush_task) +{ + return PutAllImpl(flush_task, nullptr, nullptr); +} + +bool DataStoreServiceClient::PutAllImpl( + std::unordered_map>> + &flush_task, + const std::function *yield_fptr, + const std::function *resume_fptr) { DLOG(INFO) << "DataStoreServiceClient::PutAll called with " << flush_task.size() << " tables to flush."; @@ -433,6 +443,12 @@ bool DataStoreServiceClient::PutAll( // Set up global coordinator sync_putall->total_partitions_ = sync_putall->partition_states_.size(); + // Set coroutine callbacks BEFORE starting async work (see plan risk analysis) + if (yield_fptr != nullptr && resume_fptr != nullptr) + { + sync_putall->SetCoroCallbacks(yield_fptr, resume_fptr); + } + // Start concurrent processing for each partition for (size_t i = 0; i < callback_data_list.size(); ++i) { @@ -467,13 +483,13 @@ bool DataStoreServiceClient::PutAll( } // Wait for all partitions to complete + if (yield_fptr != nullptr && resume_fptr != nullptr) { - std::unique_lock lk(sync_putall->mux_); - while (sync_putall->completed_partitions_ < - sync_putall->total_partitions_) - { - sync_putall->cv_.wait(lk); - } + sync_putall->Wait(yield_fptr, resume_fptr); + } + else + { + sync_putall->Wait(); } // Check for errors diff --git a/store_handler/data_store_service_client.h b/store_handler/data_store_service_client.h index e086a17f..fda211fa 100644 --- a/store_handler/data_store_service_client.h +++ b/store_handler/data_store_service_client.h @@ -598,6 +598,13 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler bool is_range_partition) const; private: + bool PutAllImpl(std::unordered_map< + std::string_view, + std::vector>> + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr); + int32_t MapKeyHashToPartitionId(const txservice::TxKey &key) const { return txservice::Sharder::MapKeyHashToHashPartitionId(key.Hash()); diff --git a/store_handler/data_store_service_client_closure.cpp b/store_handler/data_store_service_client_closure.cpp index bdddbec3..81225f3f 100644 --- a/store_handler/data_store_service_client_closure.cpp +++ b/store_handler/data_store_service_client_closure.cpp @@ -513,6 +513,34 @@ void SyncConcurrentRequestCallback(void *data, callback_data->Finish(result); } +void SyncPutAllData::Wait() +{ + std::unique_lock lk(mux_); + while (completed_partitions_ < total_partitions_) + { + cv_.wait(lk); + } +} + +void SyncPutAllData::Wait(const std::function *yield_fn, + const std::function *resume_fn) +{ + if (yield_fn == nullptr || resume_fn == nullptr) + { + Wait(); + return; + } + std::unique_lock lk(mux_); + while (completed_partitions_ < total_partitions_) + { + waiting_.store(true, std::memory_order_release); + lk.unlock(); + (*yield_fn)(); + lk.lock(); + waiting_.store(false, std::memory_order_release); + } +} + void PartitionBatchCallback(void *data, ::google::protobuf::Closure *closure, DataStoreServiceClient &client, diff --git a/store_handler/data_store_service_client_closure.h b/store_handler/data_store_service_client_closure.h index b8c3813c..7af9e02c 100644 --- a/store_handler/data_store_service_client_closure.h +++ b/store_handler/data_store_service_client_closure.h @@ -25,6 +25,8 @@ #include #include +#include +#include #include #include #include @@ -215,12 +217,18 @@ struct SyncPutAllData : public Poolable partition_states_.clear(); completed_partitions_ = 0; total_partitions_ = 0; + waiting_.store(false); + yield_fn_ = nullptr; + resume_fn_ = nullptr; } virtual void Clear() override { completed_partitions_ = 0; total_partitions_ = 0; + waiting_.store(false); + yield_fn_ = nullptr; + resume_fn_ = nullptr; for (auto *partition_state : partition_states_) { partition_state->Clear(); @@ -228,15 +236,41 @@ struct SyncPutAllData : public Poolable } partition_states_.clear(); } + + void SetCoroCallbacks(const std::function *yield_fn, + const std::function *resume_fn) + { + yield_fn_ = yield_fn; + resume_fn_ = resume_fn; + } + + void Wait(); + + void Wait(const std::function *yield_fn, + const std::function *resume_fn); + void OnPartitionCompleted() { std::unique_lock lk(mux_); completed_partitions_++; if (completed_partitions_ >= total_partitions_) { - cv_.notify_one(); + if (resume_fn_ && waiting_.load(std::memory_order_acquire)) + { + waiting_.store(false, std::memory_order_release); + lk.unlock(); + (*resume_fn_)(); + // Do not re-lock: resume_fn may acquire flush_worker_mux; + // coroutine will need mux_ when it resumes. Unlock before + // resume_fn avoids deadlock. + } + else if (!resume_fn_) + { + cv_.notify_one(); + } } } + mutable bthread::Mutex mux_; bthread::ConditionVariable cv_; @@ -244,6 +278,11 @@ struct SyncPutAllData : public Poolable std::vector partition_states_; int32_t completed_partitions_{0}; int32_t total_partitions_{0}; + + // Coroutine yield/resume support + const std::function *yield_fn_{nullptr}; + const std::function *resume_fn_{nullptr}; + std::atomic waiting_{false}; }; /** diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index da8fb7e8..5e0e4799 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -23,6 +23,9 @@ #include #include +#include +#include +#include #include #include #include @@ -63,10 +66,6 @@ #include "tx_start_ts_collector.h" #include "type.h" -#include -#include -#include - #ifdef WITH_JEMALLOC // we use uint32_t instead of unsigned to make the code more readable static_assert(std::is_same_v, @@ -95,7 +94,7 @@ struct CoroCtx std::unique_ptr task_; std::function sync_yield_func; // 同步 yield：入队后让出 std::function yield_fn; // 异步 yield - std::function resume_fn; // 用 weak_ptr 捕获 ctx，无参调用 + std::function resume_fn; // 用 weak_ptr 捕获 ctx，无参调用 }; struct DataMigrationStatus @@ -2524,10 +2523,10 @@ class LocalCcShards #ifndef NDEBUG boost::context::protected_fixedsize_stack flush_coro_stack_allocator_{ - 256 * 1024}; // 256KB, guard page for stack overflow detection + 512 * 1024}; // 512KB, guard page for stack overflow detection #else boost::context::pooled_fixedsize_stack flush_coro_stack_allocator_{ - 256 * 1024}; // 256KB for FlushData call chain + 1024 * 1024}; // 512KB for FlushData call chain #endif void FlushDataWorker(size_t worker_idx); @@ -2535,12 +2534,11 @@ class LocalCcShards size_t worker_idx); bool ShouldYieldFlushData(size_t worker_idx) const; - void FlushDataImpl( - FlushDataTask *cur_work, - size_t worker_idx, - const std::function &sync_yield_func, - const std::function &yield_fn, - const std::function &resume_fn); + void FlushDataImpl(FlushDataTask *cur_work, + size_t worker_idx, + const std::function &sync_yield_func, + const std::function &yield_fn, + const std::function &resume_fn); // Memory controller for data sync. DataSyncMemoryController data_sync_mem_controller_; From 21a05d71ffabf61798ff5bc4c015b2454b34e8ad Mon Sep 17 00:00:00 2001 From: lokax Date: Thu, 26 Feb 2026 16:41:06 +0800 Subject: [PATCH 06/38] phase2 --- store_handler/data_store_service_client.cpp | 170 ++++-------------- store_handler/data_store_service_client.h | 9 + .../data_store_service_client_closure.cpp | 42 +++++ .../data_store_service_client_closure.h | 34 +++- 4 files changed, 115 insertions(+), 140 deletions(-) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index 7a2fddd0..d187d10a 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -1546,16 +1546,7 @@ void DataStoreServiceClient::DispatchRangeSliceBatches( keys.size() > 0) { // Concurrency control: wait if limit reached, then increment - // counter - { - std::unique_lock lk(sync_concurrent->mux_); - while (sync_concurrent->unfinished_request_cnt_ >= - SyncConcurrentRequest::max_flying_write_count) - { - sync_concurrent->cv_.wait(lk); - } - sync_concurrent->unfinished_request_cnt_++; - } + sync_concurrent->WaitForCapacityAndIncrement(); // Dispatch current batch BatchWriteRecords(kv_table_name, @@ -1600,15 +1591,7 @@ void DataStoreServiceClient::DispatchRangeSliceBatches( if (keys.size() > 0) { // Concurrency control: wait if limit reached, then increment counter - { - std::unique_lock lk(sync_concurrent->mux_); - while (sync_concurrent->unfinished_request_cnt_ >= - SyncConcurrentRequest::max_flying_write_count) - { - sync_concurrent->cv_.wait(lk); - } - sync_concurrent->unfinished_request_cnt_++; - } + sync_concurrent->WaitForCapacityAndIncrement(); BatchWriteRecords(kv_table_name, kv_partition_id, @@ -1710,16 +1693,7 @@ void DataStoreServiceClient::DispatchRangeMetadataBatches( keys.size() > 0) { // Concurrency control: wait if limit reached, then increment - // counter - { - std::unique_lock lk(sync_concurrent->mux_); - while (sync_concurrent->unfinished_request_cnt_ >= - SyncConcurrentRequest::max_flying_write_count) - { - sync_concurrent->cv_.wait(lk); - } - sync_concurrent->unfinished_request_cnt_++; - } + sync_concurrent->WaitForCapacityAndIncrement(); // Dispatch current batch BatchWriteRecords(target_table_name, @@ -1764,16 +1738,7 @@ void DataStoreServiceClient::DispatchRangeMetadataBatches( if (keys.size() > 0) { // Concurrency control: wait if limit reached, then increment - // counter - { - std::unique_lock lk(sync_concurrent->mux_); - while (sync_concurrent->unfinished_request_cnt_ >= - SyncConcurrentRequest::max_flying_write_count) - { - sync_concurrent->cv_.wait(lk); - } - sync_concurrent->unfinished_request_cnt_++; - } + sync_concurrent->WaitForCapacityAndIncrement(); BatchWriteRecords(target_table_name, kv_partition_id, @@ -1859,14 +1824,7 @@ bool DataStoreServiceClient::UpdateRangeSlices( } // 3- Wait for slice requests to complete - { - std::unique_lock lk(slice_sync_concurrent->mux_); - slice_sync_concurrent->all_request_started_ = true; - while (slice_sync_concurrent->unfinished_request_cnt_ != 0) - { - slice_sync_concurrent->cv_.wait(lk); - } - } + slice_sync_concurrent->WaitForAll(); if (slice_sync_concurrent->result_.error_code() != remote::DataStoreError::NO_ERROR) @@ -1906,14 +1864,7 @@ bool DataStoreServiceClient::UpdateRangeSlices( kv_range_table_name, meta_acc, meta_sync_concurrent); // 5- Wait for metadata requests to complete - { - std::unique_lock lk(meta_sync_concurrent->mux_); - meta_sync_concurrent->all_request_started_ = true; - while (meta_sync_concurrent->unfinished_request_cnt_ != 0) - { - meta_sync_concurrent->cv_.wait(lk); - } - } + meta_sync_concurrent->WaitForAll(); // 6- Check for errors if (meta_sync_concurrent->result_.error_code() != @@ -1994,14 +1945,7 @@ bool DataStoreServiceClient::UpdateRangeSlices( slice_sync_concurrent); // 3- Wait for slice requests to complete. Make sure meta data is updated // after all slice info is written. - { - std::unique_lock lk(slice_sync_concurrent->mux_); - slice_sync_concurrent->all_request_started_ = true; - while (slice_sync_concurrent->unfinished_request_cnt_ != 0) - { - slice_sync_concurrent->cv_.wait(lk); - } - } + slice_sync_concurrent->WaitForAll(); if (slice_sync_concurrent->result_.error_code() != remote::DataStoreError::NO_ERROR) @@ -2052,14 +1996,7 @@ bool DataStoreServiceClient::UpdateRangeSlices( kv_range_table_name, meta_acc, meta_sync_concurrent); // 5- Wait for metadata requests to complete - { - std::unique_lock lk(meta_sync_concurrent->mux_); - meta_sync_concurrent->all_request_started_ = true; - while (meta_sync_concurrent->unfinished_request_cnt_ != 0) - { - meta_sync_concurrent->cv_.wait(lk); - } - } + meta_sync_concurrent->WaitForAll(); // 6- Check for errors if (meta_sync_concurrent->result_.error_code() != @@ -2178,14 +2115,7 @@ bool DataStoreServiceClient::UpsertRanges( } // 3- Wait for slice requests to complete - { - std::unique_lock lk(slice_sync_concurrent->mux_); - slice_sync_concurrent->all_request_started_ = true; - while (slice_sync_concurrent->unfinished_request_cnt_ != 0) - { - slice_sync_concurrent->cv_.wait(lk); - } - } + slice_sync_concurrent->WaitForAll(); if (slice_sync_concurrent->result_.error_code() != remote::DataStoreError::NO_ERROR) { @@ -2224,14 +2154,7 @@ bool DataStoreServiceClient::UpsertRanges( kv_range_table_name, meta_acc, meta_sync_concurrent); // 5- Wait for metadata requests to complete - { - std::unique_lock lk(meta_sync_concurrent->mux_); - meta_sync_concurrent->all_request_started_ = true; - while (meta_sync_concurrent->unfinished_request_cnt_ != 0) - { - meta_sync_concurrent->cv_.wait(lk); - } - } + meta_sync_concurrent->WaitForAll(); // 6- Check for errors if (meta_sync_concurrent->result_.error_code() != @@ -2926,6 +2849,16 @@ bool DataStoreServiceClient::PutArchivesAll( std::unordered_map>> &flush_task) +{ + return PutArchivesAllImpl(flush_task, nullptr, nullptr); +} + +bool DataStoreServiceClient::PutArchivesAllImpl( + std::unordered_map>> + &flush_task, + const std::function *yield_fptr, + const std::function *resume_fptr) { std::unordered_map< uint32_t, @@ -2988,6 +2921,11 @@ bool DataStoreServiceClient::PutArchivesAll( PoolableGuard guard(sync_concurrent); sync_concurrent->Reset(); + if (yield_fptr != nullptr && resume_fptr != nullptr) + { + sync_concurrent->SetCoroCallbacks(yield_fptr, resume_fptr); + } + size_t recs_cnt = archive_ptrs.size(); keys.reserve(recs_cnt * parts_cnt_per_key); records.reserve(recs_cnt * parts_cnt_per_record); @@ -3003,15 +2941,7 @@ bool DataStoreServiceClient::PutArchivesAll( if (write_batch_size >= MAX_WRITE_BATCH_SIZE) { // Wait for in-flight requests to decrease if limit reached - { - std::unique_lock lk(sync_concurrent->mux_); - while (sync_concurrent->unfinished_request_cnt_ >= - SyncConcurrentRequest::max_flying_write_count) - { - sync_concurrent->cv_.wait(lk); - } - sync_concurrent->unfinished_request_cnt_++; - } + sync_concurrent->WaitForCapacityAndIncrement(); BatchWriteRecords(kv_mvcc_archive_name, partition_id, data_shard_id, @@ -3121,14 +3051,7 @@ bool DataStoreServiceClient::PutArchivesAll( } // Wait the result. - { - std::unique_lock lk(sync_concurrent->mux_); - sync_concurrent->all_request_started_ = true; - while (sync_concurrent->unfinished_request_cnt_ != 0) - { - sync_concurrent->cv_.wait(lk); - } - } + sync_concurrent->WaitForAll(); if (sync_concurrent->result_.error_code() != remote::DataStoreError::NO_ERROR) @@ -4953,14 +4876,7 @@ bool DataStoreServiceClient::DeleteTableRanges( for (uint32_t kv_partition_id = 0; kv_partition_id < kv_partition_cnt; ++kv_partition_id) { - std::unique_lock lk( - delete_slices_sync_concurrent->mux_); - while (delete_slices_sync_concurrent->unfinished_request_cnt_ >= - SyncConcurrentRequest::max_flying_write_count) - { - delete_slices_sync_concurrent->cv_.wait(lk); - } - delete_slices_sync_concurrent->unfinished_request_cnt_++; + delete_slices_sync_concurrent->WaitForCapacityAndIncrement(); // get shard id uint32_t data_shard_id = @@ -4976,16 +4892,7 @@ bool DataStoreServiceClient::DeleteTableRanges( SyncConcurrentRequestCallback); } - // callback_data->Wait(); - { - std::unique_lock lk( - delete_slices_sync_concurrent->mux_); - delete_slices_sync_concurrent->all_request_started_ = true; - while (delete_slices_sync_concurrent->unfinished_request_cnt_ != 0) - { - delete_slices_sync_concurrent->cv_.wait(lk); - } - } + delete_slices_sync_concurrent->WaitForAll(); if (delete_slices_sync_concurrent->result_.error_code() != EloqDS::remote::DataStoreError::NO_ERROR) @@ -5001,14 +4908,7 @@ bool DataStoreServiceClient::DeleteTableRanges( for (int32_t kv_partition_id = 0; kv_partition_id < kv_partition_cnt; ++kv_partition_id) { - std::unique_lock lk( - delete_slices_sync_concurrent->mux_); - while (delete_slices_sync_concurrent->unfinished_request_cnt_ >= - SyncConcurrentRequest::max_flying_write_count) - { - delete_slices_sync_concurrent->cv_.wait(lk); - } - delete_slices_sync_concurrent->unfinished_request_cnt_++; + delete_slices_sync_concurrent->WaitForCapacityAndIncrement(); // get shard id uint32_t data_shard_id = @@ -5023,15 +4923,7 @@ bool DataStoreServiceClient::DeleteTableRanges( SyncConcurrentRequestCallback); } - { - std::unique_lock lk( - delete_slices_sync_concurrent->mux_); - delete_slices_sync_concurrent->all_request_started_ = true; - while (delete_slices_sync_concurrent->unfinished_request_cnt_ != 0) - { - delete_slices_sync_concurrent->cv_.wait(lk); - } - } + delete_slices_sync_concurrent->WaitForAll(); if (delete_slices_sync_concurrent->result_.error_code() != EloqDS::remote::DataStoreError::NO_ERROR) diff --git a/store_handler/data_store_service_client.h b/store_handler/data_store_service_client.h index fda211fa..53ecbbdc 100644 --- a/store_handler/data_store_service_client.h +++ b/store_handler/data_store_service_client.h @@ -427,6 +427,7 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler std::string_view, std::vector>> &flush_task) override; + /** * @brief Copy record from base/sk table to mvcc_archives. */ @@ -598,6 +599,14 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler bool is_range_partition) const; private: + bool PutArchivesAllImpl(std::unordered_map< + std::string_view, + std::vector< + std::unique_ptr>> + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr); + bool PutAllImpl(std::unordered_map< std::string_view, std::vector>> diff --git a/store_handler/data_store_service_client_closure.cpp b/store_handler/data_store_service_client_closure.cpp index 81225f3f..cdb747da 100644 --- a/store_handler/data_store_service_client_closure.cpp +++ b/store_handler/data_store_service_client_closure.cpp @@ -541,6 +541,48 @@ void SyncPutAllData::Wait(const std::function *yield_fn, } } +void SyncConcurrentRequest::WaitForCapacityAndIncrement() +{ + std::unique_lock lk(mux_); + while (unfinished_request_cnt_ >= max_flying_write_count) + { + if (yield_fn_ != nullptr && resume_fn_ != nullptr) + { + waiting_.store(true, std::memory_order_release); + lk.unlock(); + (*yield_fn_)(); + lk.lock(); + waiting_.store(false, std::memory_order_release); + } + else + { + cv_.wait(lk); + } + } + unfinished_request_cnt_++; +} + +void SyncConcurrentRequest::WaitForAll() +{ + std::unique_lock lk(mux_); + all_request_started_ = true; + while (unfinished_request_cnt_ != 0) + { + if (yield_fn_ != nullptr && resume_fn_ != nullptr) + { + waiting_.store(true, std::memory_order_release); + lk.unlock(); + (*yield_fn_)(); + lk.lock(); + waiting_.store(false, std::memory_order_release); + } + else + { + cv_.wait(lk); + } + } +} + void PartitionBatchCallback(void *data, ::google::protobuf::Closure *closure, DataStoreServiceClient &client, diff --git a/store_handler/data_store_service_client_closure.h b/store_handler/data_store_service_client_closure.h index 7af9e02c..0eb35438 100644 --- a/store_handler/data_store_service_client_closure.h +++ b/store_handler/data_store_service_client_closure.h @@ -312,6 +312,9 @@ struct SyncConcurrentRequest : public Poolable unfinished_request_cnt_ = 0; all_request_started_ = false; result_.Clear(); + waiting_.store(false); + yield_fn_ = nullptr; + resume_fn_ = nullptr; } virtual void Clear() override @@ -319,8 +322,22 @@ struct SyncConcurrentRequest : public Poolable unfinished_request_cnt_ = 0; all_request_started_ = false; result_.Clear(); + waiting_.store(false); + yield_fn_ = nullptr; + resume_fn_ = nullptr; + } + + void SetCoroCallbacks(const std::function *yield_fn, + const std::function *resume_fn) + { + yield_fn_ = yield_fn; + resume_fn_ = resume_fn; } + void WaitForCapacityAndIncrement(); + + void WaitForAll(); + void Finish(const remote::CommonResult &res) { std::unique_lock lk(mux_); @@ -334,7 +351,17 @@ struct SyncConcurrentRequest : public Poolable if ((all_request_started_ && unfinished_request_cnt_ == 0) || unfinished_request_cnt_ == max_flying_write_count - 1) { - cv_.notify_one(); + if (resume_fn_ && waiting_.load(std::memory_order_acquire)) + { + waiting_.store(false, std::memory_order_release); + lk.unlock(); + (*resume_fn_)(); + // Do not re-lock: resume_fn may acquire flush_worker_mux + } + else if (!resume_fn_) + { + cv_.notify_one(); + } } } @@ -342,6 +369,11 @@ struct SyncConcurrentRequest : public Poolable int32_t unfinished_request_cnt_{0}; bool all_request_started_{false}; remote::CommonResult result_; + + // Coroutine yield/resume support + const std::function *yield_fn_{nullptr}; + const std::function *resume_fn_{nullptr}; + std::atomic waiting_{false}; mutable bthread::Mutex mux_; bthread::ConditionVariable cv_; }; From c9dfe4fbc5e43713602aa5ec37f986e56a77da4d Mon Sep 17 00:00:00 2001 From: lokax Date: Thu, 26 Feb 2026 17:11:12 +0800 Subject: [PATCH 07/38] phase4 --- store_handler/data_store_service_client.cpp | 45 ++++-- store_handler/data_store_service_client.h | 22 ++- .../data_store_service_client_closure.h | 52 ++++++- store_handler/rocksdb_handler.cpp | 18 ++- store_handler/rocksdb_handler.h | 15 +- tx_service/include/store/data_store_handler.h | 19 ++- tx_service/include/store/int_mem_store.h | 19 ++- tx_service/src/cc/local_cc_shards.cpp | 130 +++++++++++------- tx_service/src/checkpointer.cpp | 4 +- 9 files changed, 239 insertions(+), 85 deletions(-) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index d187d10a..a639ee71 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -313,9 +313,11 @@ void DataStoreServiceClient::ScheduleTimerTasks() bool DataStoreServiceClient::PutAll( std::unordered_map>> - &flush_task) + &flush_task, + const std::function *yield_fptr, + const std::function *resume_fptr) { - return PutAllImpl(flush_task, nullptr, nullptr); + return PutAllImpl(flush_task, yield_fptr, resume_fptr); } bool DataStoreServiceClient::PutAllImpl( @@ -2848,9 +2850,11 @@ void DataStoreServiceClient::DecodeArchiveValue( bool DataStoreServiceClient::PutArchivesAll( std::unordered_map>> - &flush_task) + &flush_task, + const std::function *yield_fptr, + const std::function *resume_fptr) { - return PutArchivesAllImpl(flush_task, nullptr, nullptr); + return PutArchivesAllImpl(flush_task, yield_fptr, resume_fptr); } bool DataStoreServiceClient::PutArchivesAllImpl( @@ -3087,7 +3091,19 @@ bool DataStoreServiceClient::PutArchivesAllImpl( bool DataStoreServiceClient::CopyBaseToArchive( std::unordered_map>> - &flush_task) + &flush_task, + const std::function *yield_fptr, + const std::function *resume_fptr) +{ + return CopyBaseToArchiveImpl(flush_task, yield_fptr, resume_fptr); +} + +bool DataStoreServiceClient::CopyBaseToArchiveImpl( + std::unordered_map>> + &flush_task, + const std::function *yield_fptr, + const std::function *resume_fptr) { // Prepare for the copied base table data to be flushed to the archive table std::unordered_map= MAX_FLYING_READ_COUNT) { - callback_data->Wait(); + if (yield_fptr && resume_fptr) + { + callback_data->Wait(yield_fptr, resume_fptr); + } + else + { + callback_data->Wait(); + } } if (callback_data->GetErrorCode() != 0) { @@ -3163,6 +3187,11 @@ bool DataStoreServiceClient::CopyBaseToArchive( } // Wait the result all return before returning to avoid referencing // invalid memory in callback. + if (yield_fptr && resume_fptr) + { + callback_datas[0].Wait(yield_fptr, resume_fptr); + } + else { std::unique_lock lk(mtx); while (flying_cnt > 0) @@ -3261,7 +3290,7 @@ bool DataStoreServiceClient::CopyBaseToArchive( { // Put the archive records to the archive table. // This is a sync call - bool ret = PutArchivesAll(archive_flush_task); + bool ret = PutArchivesAll(archive_flush_task, yield_fptr, resume_fptr); if (!ret) { return false; diff --git a/store_handler/data_store_service_client.h b/store_handler/data_store_service_client.h index 53ecbbdc..1cbc63d7 100644 --- a/store_handler/data_store_service_client.h +++ b/store_handler/data_store_service_client.h @@ -232,7 +232,9 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler bool PutAll(std::unordered_map< std::string_view, std::vector>> - &flush_task) override; + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) override; bool NeedPersistKV() override { @@ -426,7 +428,10 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler bool PutArchivesAll(std::unordered_map< std::string_view, std::vector>> - &flush_task) override; + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) + override; /** * @brief Copy record from base/sk table to mvcc_archives. @@ -435,7 +440,9 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler std::unordered_map< std::string_view, std::vector>> - &flush_task) override; + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) override; /** * @brief Get the latest visible(commit_ts <= upper_bound_ts) @@ -614,6 +621,15 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler const std::function *yield_fptr = nullptr, const std::function *resume_fptr = nullptr); + bool CopyBaseToArchiveImpl(std::unordered_map< + std::string_view, + std::vector< + std::unique_ptr< + txservice::FlushTaskEntry>>> + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr); + int32_t MapKeyHashToPartitionId(const txservice::TxKey &key) const { return txservice::Sharder::MapKeyHashToHashPartitionId(key.Hash()); diff --git a/store_handler/data_store_service_client_closure.h b/store_handler/data_store_service_client_closure.h index 0eb35438..41ee8597 100644 --- a/store_handler/data_store_service_client_closure.h +++ b/store_handler/data_store_service_client_closure.h @@ -238,7 +238,7 @@ struct SyncPutAllData : public Poolable } void SetCoroCallbacks(const std::function *yield_fn, - const std::function *resume_fn) + const std::function *resume_fn) { yield_fn_ = yield_fn; resume_fn_ = resume_fn; @@ -328,7 +328,7 @@ struct SyncConcurrentRequest : public Poolable } void SetCoroCallbacks(const std::function *yield_fn, - const std::function *resume_fn) + const std::function *resume_fn) { yield_fn_ = yield_fn; resume_fn_ = resume_fn; @@ -520,11 +520,14 @@ struct ReadBaseForArchiveCallbackData ReadBaseForArchiveCallbackData(bthread::Mutex &mtx, bthread::ConditionVariable &cv, size_t &flying_read_cnt, - int &error_code) + int &error_code, + const std::function *resume_fn = + nullptr) : mtx_(mtx), cv_(cv), flying_read_cnt_(flying_read_cnt), error_code_(error_code), + resume_fn_(resume_fn), partition_id_(0), key_str_(), value_str_(), @@ -533,6 +536,11 @@ struct ReadBaseForArchiveCallbackData { } + void SetCoroCallbacks(const std::function *resume_fn) + { + resume_fn_ = resume_fn; + } + void ResetResult() { partition_id_ = 0; @@ -551,6 +559,30 @@ struct ReadBaseForArchiveCallbackData } } + void Wait(const std::function *yield_fn, + const std::function *resume_fn) + { + if (!yield_fn || !resume_fn) + { + Wait(); + return; + } + std::unique_lock lk(mtx_); + resume_fn_ = resume_fn; + while (flying_read_cnt_ > 0) + { + waiting_.store(true, std::memory_order_release); + lk.unlock(); + (*yield_fn)(); + lk.lock(); + waiting_.store(false, std::memory_order_release); + while (flying_read_cnt_ > 0) + { + cv_.wait(lk); + } + } + } + size_t AddFlyingReadCount() { std::unique_lock lk(mtx_); @@ -564,7 +596,17 @@ struct ReadBaseForArchiveCallbackData flying_read_cnt_--; if (flying_read_cnt_ == 0) { - cv_.notify_one(); + if (resume_fn_ && waiting_.load(std::memory_order_acquire)) + { + waiting_.store(false, std::memory_order_release); + lk.unlock(); + (*resume_fn_)(); + // Do not re-lock: resume_fn may acquire flush_worker_mux + } + else + { + cv_.notify_one(); + } } return flying_read_cnt_; } @@ -608,6 +650,8 @@ struct ReadBaseForArchiveCallbackData bthread::ConditionVariable &cv_; size_t &flying_read_cnt_; int &error_code_; + const std::function *resume_fn_{nullptr}; + std::atomic waiting_{false}; int32_t partition_id_; std::string_view key_str_; std::string value_str_; diff --git a/store_handler/rocksdb_handler.cpp b/store_handler/rocksdb_handler.cpp index fb81f021..b6a0ff5c 100644 --- a/store_handler/rocksdb_handler.cpp +++ b/store_handler/rocksdb_handler.cpp @@ -433,8 +433,12 @@ void RocksDBHandler::DeserializeRecord(const char *payload, bool RocksDBHandler::PutAll( std::unordered_map>> - &batch) + &batch, + const std::function *yield_fptr, + const std::function *resume_fptr) { + (void)yield_fptr; + (void)resume_fptr; std::thread::id this_id = std::this_thread::get_id(); if (batch.empty()) { @@ -1663,8 +1667,12 @@ std::string RocksDBHandler::CreateNewKVCatalogInfo( bool RocksDBHandler::PutArchivesAll( std::unordered_map>> - &batch) + &batch, + const std::function *yield_fptr, + const std::function *resume_fptr) { + (void)yield_fptr; + (void)resume_fptr; LOG(ERROR) << "RocksDBHandler::PutArchivesAll not implemented"; // Not implemented assert(false); @@ -1676,8 +1684,12 @@ bool RocksDBHandler::PutArchivesAll( bool RocksDBHandler::CopyBaseToArchive( std::unordered_map>> - &batch) + &batch, + const std::function *yield_fptr, + const std::function *resume_fptr) { + (void)yield_fptr; + (void)resume_fptr; LOG(ERROR) << "RocksDBHandler::CopyBaseToArchive not implemented"; // Not implemented assert(false); diff --git a/store_handler/rocksdb_handler.h b/store_handler/rocksdb_handler.h index ed441153..0f63c95e 100644 --- a/store_handler/rocksdb_handler.h +++ b/store_handler/rocksdb_handler.h @@ -274,8 +274,9 @@ class RocksDBHandler : public txservice::store::DataStoreHandler */ bool PutAll(std::unordered_map< std::string_view, - std::vector>> &batch) - override; + std::vector>> &batch, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) override; /** * @brief indicate end of flush entries in a single ckpt for \@param @@ -480,15 +481,19 @@ class RocksDBHandler : public txservice::store::DataStoreHandler bool PutArchivesAll(std::unordered_map< std::string_view, std::vector>> - &batch) override; + &batch, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) + override; /** * @brief Copy record from base/sk table to mvcc_archives. */ bool CopyBaseToArchive( std::unordered_map< std::string_view, - std::vector>> &batch) - override; + std::vector>> &batch, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) override; /** * @brief Get the latest visible(commit_ts <= upper_bound_ts) diff --git a/tx_service/include/store/data_store_handler.h b/tx_service/include/store/data_store_handler.h index abb7b0ab..dcee8691 100644 --- a/tx_service/include/store/data_store_handler.h +++ b/tx_service/include/store/data_store_handler.h @@ -84,10 +84,13 @@ class DataStoreHandler * data_sync_vec_ in each flush task entry. * @return whether all entries are written to data store successfully */ - virtual bool PutAll(std::unordered_map< - std::string_view, - std::vector>> - &flush_task) = 0; + virtual bool PutAll( + std::unordered_map< + std::string_view, + std::vector>> + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) = 0; /** * @brief indicate end of flush entries in a single ckpt for \@param batch @@ -285,7 +288,9 @@ class DataStoreHandler std::unordered_map< std::string_view, std::vector>> - &flush_task) = 0; + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) = 0; /** * @brief Copy record from base/sk table to mvcc_archives. */ @@ -293,7 +298,9 @@ class DataStoreHandler std::unordered_map< std::string_view, std::vector>> - &flush_task) = 0; + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) = 0; /** * @brief Get the latest visible(commit_ts <= upper_bound_ts) historical diff --git a/tx_service/include/store/int_mem_store.h b/tx_service/include/store/int_mem_store.h index eb3dffa6..ae854959 100644 --- a/tx_service/include/store/int_mem_store.h +++ b/tx_service/include/store/int_mem_store.h @@ -57,8 +57,12 @@ class IntMemoryStore : public DataStoreHandler bool PutAll(std::unordered_map< std::string_view, std::vector>> - &flush_task) override + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) override { + (void)yield_fptr; + (void)resume_fptr; assert(false); // for (const auto &ref : batch) // { @@ -246,8 +250,13 @@ class IntMemoryStore : public DataStoreHandler bool PutArchivesAll(std::unordered_map< std::string_view, std::vector>> - &flush_task) override + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) + override { + (void)yield_fptr; + (void)resume_fptr; assert(false); return true; } @@ -258,8 +267,12 @@ class IntMemoryStore : public DataStoreHandler std::unordered_map< std::string_view, std::vector>> - &flush_task) override + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) override { + (void)yield_fptr; + (void)resume_fptr; assert(false); return true; } diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 157c3e66..e27e17de 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -5854,26 +5854,19 @@ bool LocalCcShards::ShouldYieldFlushData(size_t worker_idx) const return !pending_flush_work_[worker_idx].empty(); } -void LocalCcShards::FlushDataImpl( - FlushDataTask *cur_work, - size_t worker_idx, - const std::function &sync_yield_func, - const std::function &yield_fn, - const std::function &resume_fn) +void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, + size_t worker_idx, + const std::function &sync_yield_func, + const std::function &yield_fn, + const std::function &resume_fn) { - (void)yield_fn; - (void)resume_fn; - auto &flush_task_entries = cur_work->flush_task_entries_; bool succ = true; - if (ShouldYieldFlushData(worker_idx)) - { - sync_yield_func(); - } if (EnableMvcc()) { - succ = store_hd_->CopyBaseToArchive(flush_task_entries); + succ = store_hd_->CopyBaseToArchive( + flush_task_entries, &yield_fn, &resume_fn); if (!succ) { LOG(ERROR) << "DataSync CopyBaseToArchive flush to kv " @@ -5881,13 +5874,9 @@ void LocalCcShards::FlushDataImpl( } } - if (ShouldYieldFlushData(worker_idx)) - { - sync_yield_func(); - } if (succ) { - succ = store_hd_->PutAll(flush_task_entries); + succ = store_hd_->PutAll(flush_task_entries, &yield_fn, &resume_fn); if (!succ) { LOG(ERROR) << "DataSync PutAll flush to kv " @@ -5895,13 +5884,10 @@ void LocalCcShards::FlushDataImpl( } } - if (ShouldYieldFlushData(worker_idx)) - { - sync_yield_func(); - } if (succ && EnableMvcc()) { - succ = store_hd_->PutArchivesAll(flush_task_entries); + succ = store_hd_->PutArchivesAll( + flush_task_entries, &yield_fn, &resume_fn); if (!succ) { LOG(ERROR) << "DataSync PutArchivesAll flush to " @@ -5909,10 +5895,6 @@ void LocalCcShards::FlushDataImpl( } } - if (ShouldYieldFlushData(worker_idx)) - { - sync_yield_func(); - } if (succ && store_hd_->NeedPersistKV()) { std::vector kv_table_names; @@ -6014,10 +5996,6 @@ void LocalCcShards::FlushDataImpl( } } - if (ShouldYieldFlushData(worker_idx)) - { - sync_yield_func(); - } // Notify cc shards that dirty data has been flushed. This will re-enqueue // kickout data cc reqs if there are any. WaitableCc reset_cc( @@ -6093,8 +6071,7 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) // 1. 从 resume_queue 获取被唤醒的协程 if (!resume_queue.empty()) { - std::shared_ptr ctx = - std::move(resume_queue.front()); + std::shared_ptr ctx = std::move(resume_queue.front()); resume_queue.pop_front(); flush_worker_lk.unlock(); @@ -6116,8 +6093,7 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) &previous_size_update_time] { if (!pending_flush_work.empty() || !resume_queue.empty() || - flush_data_worker_ctx_.status_ == - WorkerStatus::Terminated) + flush_data_worker_ctx_.status_ == WorkerStatus::Terminated) { return true; } @@ -6187,9 +6163,10 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) [this, ctx, worker_idx](continuation &&sink) { ctx->yield_fn = [&sink]() { sink = sink.resume(); }; - ctx->sync_yield_func = - [&sink, weak_ctx = std::weak_ptr(ctx), this, - worker_idx]() + ctx->sync_yield_func = [&sink, + weak_ctx = std::weak_ptr(ctx), + this, + worker_idx]() { if (auto c = weak_ctx.lock()) { @@ -6203,8 +6180,7 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) } }; ctx->resume_fn = - [this, worker_idx, - weak_ctx = std::weak_ptr(ctx)]() + [this, worker_idx, weak_ctx = std::weak_ptr(ctx)]() { if (auto c = weak_ctx.lock()) { @@ -6225,20 +6201,72 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) flush_worker_lk.lock(); } - // Terminate 时清空队列 - while (!resume_queue.empty()) + // Terminate 时清空队列，使用协程方式执行 + while (!resume_queue.empty() || !pending_flush_work.empty()) { - std::shared_ptr ctx = - std::move(resume_queue.front()); - resume_queue.pop_front(); + // 1. 优先处理 resume_queue 中的协程 + if (!resume_queue.empty()) + { + std::shared_ptr ctx = std::move(resume_queue.front()); + resume_queue.pop_front(); + flush_worker_lk.unlock(); + ctx->coro_ = ctx->coro_.resume(); + flush_worker_lk.lock(); + continue; + } + + // 2. pending_flush_work 中有任务，以协程方式启动 + std::unique_ptr cur_work = + std::move(pending_flush_work.front()); + pending_flush_work.pop_front(); + flush_data_worker_ctx_.cv_.notify_all(); flush_worker_lk.unlock(); - ctx->coro_ = ctx->coro_.resume(); + + auto ctx = std::make_shared(); + ctx->task_ = std::move(cur_work); + ctx->coro_ = boost::context::callcc( + std::allocator_arg, + flush_coro_stack_allocator_, + [this, ctx, worker_idx](continuation &&sink) + { + ctx->yield_fn = [&sink]() { sink = sink.resume(); }; + ctx->sync_yield_func = [&sink, + weak_ctx = std::weak_ptr(ctx), + this, + worker_idx]() + { + if (auto c = weak_ctx.lock()) + { + { + std::lock_guard lk( + flush_data_worker_ctx_.mux_); + resume_queue_[worker_idx].push_back(std::move(c)); + flush_data_worker_ctx_.cv_.notify_all(); + } + sink = sink.resume(); + } + }; + ctx->resume_fn = + [this, worker_idx, weak_ctx = std::weak_ptr(ctx)]() + { + if (auto c = weak_ctx.lock()) + { + std::lock_guard lk( + flush_data_worker_ctx_.mux_); + resume_queue_[worker_idx].push_back(std::move(c)); + flush_data_worker_ctx_.cv_.notify_all(); + } + }; + FlushDataImpl(ctx->task_.get(), + worker_idx, + ctx->sync_yield_func, + ctx->yield_fn, + ctx->resume_fn); + return std::move(sink); + }); + flush_worker_lk.lock(); } - while (!pending_flush_work.empty()) - { - FlushData(flush_worker_lk, worker_idx); - } } void LocalCcShards::RangeSplitWorker() diff --git a/tx_service/src/checkpointer.cpp b/tx_service/src/checkpointer.cpp index 8ae1ea04..1031e6b0 100644 --- a/tx_service/src/checkpointer.cpp +++ b/tx_service/src/checkpointer.cpp @@ -568,7 +568,7 @@ bool Checkpointer::CkptEntryForTest( std::vector>> &flush_task_entries) { - return store_hd_->PutAll(flush_task_entries); + return store_hd_->PutAll(flush_task_entries, nullptr, nullptr); } bool Checkpointer::FlushArchiveForTest( @@ -576,6 +576,6 @@ bool Checkpointer::FlushArchiveForTest( std::vector>> &flush_task_entries) { - return store_hd_->PutArchivesAll(flush_task_entries); + return store_hd_->PutArchivesAll(flush_task_entries, nullptr, nullptr); } } // namespace txservice From d2c27e6bf7de2ccdb5fcdf4d61f95a78099924c4 Mon Sep 17 00:00:00 2001 From: lokax Date: Thu, 26 Feb 2026 18:02:41 +0800 Subject: [PATCH 08/38] phase6 --- tx_service/include/cc/cc_req_misc.h | 103 ++++++++++++++++++++++++-- tx_service/src/cc/local_cc_shards.cpp | 6 +- 2 files changed, 102 insertions(+), 7 deletions(-) diff --git a/tx_service/include/cc/cc_req_misc.h b/tx_service/include/cc/cc_req_misc.h index eedae7e7..934efde5 100644 --- a/tx_service/include/cc/cc_req_misc.h +++ b/tx_service/include/cc/cc_req_misc.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -876,6 +877,13 @@ struct WaitableCc : public RunOnTxProcessorCc error_code_ = CcErrorCode::NO_ERROR; } + void SetCoroCallbacks(const std::function *yield_fn, + const std::function *resume_fn) + { + yield_fn_ = yield_fn; + resume_fn_ = resume_fn; + } + void Wait() { std::unique_lock lk(mux_); @@ -885,6 +893,25 @@ struct WaitableCc : public RunOnTxProcessorCc } } + void Wait(const std::function *yield_fn, + const std::function *resume_fn) + { + if (yield_fn == nullptr || resume_fn == nullptr) + { + Wait(); + return; + } + std::unique_lock lk(mux_); + while (unfinished_cnt_) + { + waiting_.store(true, std::memory_order_release); + lk.unlock(); + (*yield_fn)(); + lk.lock(); + waiting_.store(false, std::memory_order_release); + } + } + bool IsFinished() const { std::lock_guard lk(mux_); @@ -910,7 +937,17 @@ struct WaitableCc : public RunOnTxProcessorCc error_code_ = error_code; if (unfinished_cnt_ == 0) { - cv_.notify_one(); + if (resume_fn_ != nullptr && + waiting_.load(std::memory_order_acquire)) + { + waiting_.store(false, std::memory_order_release); + lk.unlock(); + (*resume_fn_)(); + } + else + { + cv_.notify_one(); + } } } @@ -922,7 +959,17 @@ struct WaitableCc : public RunOnTxProcessorCc error_code_ = CcErrorCode::NO_ERROR; if (--unfinished_cnt_ == 0) { - cv_.notify_one(); + if (resume_fn_ != nullptr && + waiting_.load(std::memory_order_acquire)) + { + waiting_.store(false, std::memory_order_release); + lk.unlock(); + (*resume_fn_)(); + } + else + { + cv_.notify_one(); + } } } return false; @@ -944,6 +991,11 @@ struct WaitableCc : public RunOnTxProcessorCc uint32_t unfinished_cnt_{0}; CcErrorCode error_code_; + + // Coroutine yield/resume support + const std::function *yield_fn_{nullptr}; + const std::function *resume_fn_{nullptr}; + std::atomic waiting_{false}; }; struct UpdateCceCkptTsCc : public CcRequestBase { @@ -990,13 +1042,30 @@ struct UpdateCceCkptTsCc : public CcRequestBase bool Execute(CcShard &ccs) override; + void SetCoroCallbacks(const std::function *yield_fn, + const std::function *resume_fn) + { + yield_fn_ = yield_fn; + resume_fn_ = resume_fn; + } + void SetFinished() { - std::lock_guard lk(mux_); + std::unique_lock lk(mux_); unfinished_core_cnt_--; if (unfinished_core_cnt_ == 0) { - cv_.notify_one(); + if (resume_fn_ != nullptr && + waiting_.load(std::memory_order_acquire)) + { + waiting_.store(false, std::memory_order_release); + lk.unlock(); + (*resume_fn_)(); + } + else + { + cv_.notify_one(); + } } } @@ -1005,7 +1074,26 @@ struct UpdateCceCkptTsCc : public CcRequestBase std::unique_lock lk(mux_); while (unfinished_core_cnt_ > 0) { - cv_.wait_for(lk, 10000); + cv_.wait_for(lk, 10000L); // timeout_us, preserve original value + } + } + + void Wait(const std::function *yield_fn, + const std::function *resume_fn) + { + if (yield_fn == nullptr || resume_fn == nullptr) + { + Wait(); + return; + } + std::unique_lock lk(mux_); + while (unfinished_core_cnt_ > 0) + { + waiting_.store(true, std::memory_order_release); + lk.unlock(); + (*yield_fn)(); + lk.lock(); + waiting_.store(false, std::memory_order_release); } } @@ -1026,6 +1114,11 @@ struct UpdateCceCkptTsCc : public CcRequestBase TableName table_name_; bthread::Mutex mux_; bthread::ConditionVariable cv_; + + // Coroutine yield/resume support + const std::function *yield_fn_{nullptr}; + const std::function *resume_fn_{nullptr}; + std::atomic waiting_{false}; }; struct WaitNoNakedBucketRefCc : public CcRequestBase diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index e27e17de..390e557a 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -5985,12 +5985,13 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, entry->data_sync_task_->node_group_term_, table_name, cce_entries_map); + update_cce_req.SetCoroCallbacks(&yield_fn, &resume_fn); for (auto &[core_idx, cce_entries] : cce_entries_map) { updated_ckpt_ts_core_ids.insert(core_idx); EnqueueToCcShard(core_idx, &update_cce_req); } - update_cce_req.Wait(); + update_cce_req.Wait(&yield_fn, &resume_fn); } } } @@ -6005,11 +6006,12 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, return true; }, updated_ckpt_ts_core_ids.size()); + reset_cc.SetCoroCallbacks(&yield_fn, &resume_fn); for (uint16_t core_idx : updated_ckpt_ts_core_ids) { EnqueueToCcShard(core_idx, &reset_cc); } - reset_cc.Wait(); + reset_cc.Wait(&yield_fn, &resume_fn); auto ckpt_err = succ ? DataSyncTask::CkptErrorCode::NO_ERROR : DataSyncTask::CkptErrorCode::FLUSH_ERROR; From 5b5a9f45cc23f7db231a11f93807034e34dd6908 Mon Sep 17 00:00:00 2001 From: lokax Date: Thu, 26 Feb 2026 18:12:37 +0800 Subject: [PATCH 09/38] phase7 --- store_handler/data_store_service_client.cpp | 17 ++++++- store_handler/data_store_service_client.h | 4 +- .../data_store_service_client_closure.h | 49 ++++++++++++++++++- store_handler/rocksdb_handler.cpp | 9 +++- store_handler/rocksdb_handler.h | 4 +- tx_service/include/store/data_store_handler.h | 6 ++- tx_service/src/cc/local_cc_shards.cpp | 6 +-- 7 files changed, 82 insertions(+), 13 deletions(-) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index a639ee71..43b0d12c 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -539,14 +539,27 @@ bool DataStoreServiceClient::PutAllImpl( * fails. */ bool DataStoreServiceClient::PersistKV( - const std::vector &kv_table_names) + const std::vector &kv_table_names, + const std::function *yield_fptr, + const std::function *resume_fptr) { SyncCallbackData *callback_data = sync_callback_data_pool_.NextObject(); PoolableGuard guard(callback_data); callback_data->Reset(); + if (yield_fptr != nullptr && resume_fptr != nullptr) + { + callback_data->SetCoroCallbacks(yield_fptr, resume_fptr); + } FlushData(kv_table_names, callback_data, &SyncCallback); - callback_data->Wait(); + if (yield_fptr != nullptr && resume_fptr != nullptr) + { + callback_data->Wait(yield_fptr, resume_fptr); + } + else + { + callback_data->Wait(); + } if (callback_data->Result().error_code() != EloqDS::remote::DataStoreError::NO_ERROR) { diff --git a/store_handler/data_store_service_client.h b/store_handler/data_store_service_client.h index 1cbc63d7..0a75c2eb 100644 --- a/store_handler/data_store_service_client.h +++ b/store_handler/data_store_service_client.h @@ -249,7 +249,9 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler * @param node_group * @return whether all entries are written to data store successfully */ - bool PersistKV(const std::vector &kv_table_names) override; + bool PersistKV(const std::vector &kv_table_names, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) override; void UpsertTable( const txservice::TableSchema *old_table_schema, diff --git a/store_handler/data_store_service_client_closure.h b/store_handler/data_store_service_client_closure.h index 41ee8597..9df82f30 100644 --- a/store_handler/data_store_service_client_closure.h +++ b/store_handler/data_store_service_client_closure.h @@ -84,19 +84,42 @@ struct SyncCallbackData : public Poolable { finished_ = false; result_.Clear(); + waiting_.store(false); + yield_fn_ = nullptr; + resume_fn_ = nullptr; } virtual void Clear() override { finished_ = false; result_.Clear(); + waiting_.store(false); + yield_fn_ = nullptr; + resume_fn_ = nullptr; + } + + void SetCoroCallbacks(const std::function *yield_fn, + const std::function *resume_fn) + { + yield_fn_ = yield_fn; + resume_fn_ = resume_fn; } virtual void Notify() { std::unique_lock lk(mtx_); finished_ = true; - cv_.notify_one(); + if (resume_fn_ != nullptr && + waiting_.load(std::memory_order_acquire)) + { + waiting_.store(false, std::memory_order_release); + lk.unlock(); + (*resume_fn_)(); + } + else + { + cv_.notify_one(); + } } virtual void Wait() @@ -108,6 +131,25 @@ struct SyncCallbackData : public Poolable } } + void Wait(const std::function *yield_fn, + const std::function *resume_fn) + { + if (yield_fn == nullptr || resume_fn == nullptr) + { + Wait(); + return; + } + std::unique_lock lk(mtx_); + while (!finished_) + { + waiting_.store(true, std::memory_order_release); + lk.unlock(); + (*yield_fn)(); + lk.lock(); + waiting_.store(false, std::memory_order_release); + } + } + remote::CommonResult &Result() { return result_; @@ -125,6 +167,11 @@ struct SyncCallbackData : public Poolable bool finished_; remote::CommonResult result_; + + // Coroutine yield/resume support + const std::function *yield_fn_{nullptr}; + const std::function *resume_fn_{nullptr}; + std::atomic waiting_{false}; }; /** diff --git a/store_handler/rocksdb_handler.cpp b/store_handler/rocksdb_handler.cpp index b6a0ff5c..7c854e81 100644 --- a/store_handler/rocksdb_handler.cpp +++ b/store_handler/rocksdb_handler.cpp @@ -549,8 +549,13 @@ bool RocksDBHandler::PutAll( return true; } -bool RocksDBHandler::PersistKV(const std::vector &kv_table_names) +bool RocksDBHandler::PersistKV( + const std::vector &kv_table_names, + const std::function *yield_fptr, + const std::function *resume_fptr) { + (void)yield_fptr; + (void)resume_fptr; std::shared_lock db_lk(db_mux_); auto db = GetDBPtr(); if (!db) @@ -582,7 +587,7 @@ bool RocksDBHandler::PersistKV(const std::vector &kv_table_names) } return true; -}; +} void RocksDBHandler::UpsertTableInternal( txservice::TableEngine table_engine, diff --git a/store_handler/rocksdb_handler.h b/store_handler/rocksdb_handler.h index 0f63c95e..37964d16 100644 --- a/store_handler/rocksdb_handler.h +++ b/store_handler/rocksdb_handler.h @@ -286,7 +286,9 @@ class RocksDBHandler : public txservice::store::DataStoreHandler * @param node_group * @return whether all entries are written to data store successfully */ - bool PersistKV(const std::vector &kv_table_names) override; + bool PersistKV(const std::vector &kv_table_names, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) override; bool NeedPersistKV() override { diff --git a/tx_service/include/store/data_store_handler.h b/tx_service/include/store/data_store_handler.h index dcee8691..ee6ee9f9 100644 --- a/tx_service/include/store/data_store_handler.h +++ b/tx_service/include/store/data_store_handler.h @@ -100,8 +100,12 @@ class DataStoreHandler * @param node_group * @return whether all entries are written to data store successfully */ - virtual bool PersistKV(const std::vector &kv_table_names) + virtual bool PersistKV(const std::vector &kv_table_names, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) { + (void)yield_fptr; + (void)resume_fptr; return true; } diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 390e557a..e427c067 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -5902,7 +5902,7 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, { kv_table_names.push_back(table_name.data()); } - succ = store_hd_->PersistKV(kv_table_names); + succ = store_hd_->PersistKV(kv_table_names, &yield_fn, &resume_fn); } // Record that data was written in DataSyncStatus if flush succeeded. @@ -5936,10 +5936,6 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, { for (auto &[kv_table_name, entries] : flush_task_entries) { - if (ShouldYieldFlushData(worker_idx)) - { - sync_yield_func(); - } for (auto &entry : entries) { if (!entry->data_sync_task_->need_update_ckpt_ts_) From 2369b98c823f8e54c8aec39e4e673605d560a379 Mon Sep 17 00:00:00 2001 From: lokax Date: Fri, 27 Feb 2026 10:50:31 +0800 Subject: [PATCH 10/38] phase8 --- store_handler/data_store_service_client.cpp | 39 +++++++++++++++++-- store_handler/data_store_service_client.h | 5 ++- store_handler/rocksdb_handler.cpp | 6 ++- store_handler/rocksdb_handler.h | 5 ++- tx_service/include/cc/local_cc_shards.h | 9 ++++- tx_service/include/store/data_store_handler.h | 6 ++- tx_service/src/cc/local_cc_shards.cpp | 17 +++++--- 7 files changed, 73 insertions(+), 14 deletions(-) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index 43b0d12c..1b9c6717 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -1773,7 +1773,9 @@ void DataStoreServiceClient::DispatchRangeMetadataBatches( } bool DataStoreServiceClient::UpdateRangeSlices( - const std::vector &update_range_slice_reqs) + const std::vector &update_range_slice_reqs, + const std::function *yield_fptr, + const std::function *resume_fptr) { if (update_range_slice_reqs.empty()) { @@ -1829,6 +1831,10 @@ bool DataStoreServiceClient::UpdateRangeSlices( sync_concurrent_request_pool_.NextObject(); PoolableGuard slice_guard(slice_sync_concurrent); slice_sync_concurrent->Reset(); + if (yield_fptr != nullptr && resume_fptr != nullptr) + { + slice_sync_concurrent->SetCoroCallbacks(yield_fptr, resume_fptr); + } for (const auto &[kv_partition_id, slice_plans] : slice_plans) { // Call DispatchRangeSliceBatches once with all plans @@ -1856,11 +1862,23 @@ bool DataStoreServiceClient::UpdateRangeSlices( sync_callback_data_pool_.NextObject(); PoolableGuard guard(flush_slices_callback_data); flush_slices_callback_data->Reset(); + if (yield_fptr != nullptr && resume_fptr != nullptr) + { + flush_slices_callback_data->SetCoroCallbacks(yield_fptr, + resume_fptr); + } std::vector kv_slices_table_names; kv_slices_table_names.emplace_back(kv_range_slices_table_name); FlushData( kv_slices_table_names, flush_slices_callback_data, &SyncCallback); - flush_slices_callback_data->Wait(); + if (yield_fptr != nullptr && resume_fptr != nullptr) + { + flush_slices_callback_data->Wait(yield_fptr, resume_fptr); + } + else + { + flush_slices_callback_data->Wait(); + } if (flush_slices_callback_data->Result().error_code() != EloqDS::remote::DataStoreError::NO_ERROR) { @@ -1875,6 +1893,10 @@ bool DataStoreServiceClient::UpdateRangeSlices( sync_concurrent_request_pool_.NextObject(); PoolableGuard meta_guard(meta_sync_concurrent); meta_sync_concurrent->Reset(); + if (yield_fptr != nullptr && resume_fptr != nullptr) + { + meta_sync_concurrent->SetCoroCallbacks(yield_fptr, resume_fptr); + } DispatchRangeMetadataBatches( kv_range_table_name, meta_acc, meta_sync_concurrent); @@ -1896,10 +1918,21 @@ bool DataStoreServiceClient::UpdateRangeSlices( SyncCallbackData *callback_data = sync_callback_data_pool_.NextObject(); PoolableGuard guard(callback_data); callback_data->Reset(); + if (yield_fptr != nullptr && resume_fptr != nullptr) + { + callback_data->SetCoroCallbacks(yield_fptr, resume_fptr); + } std::vector kv_range_table_names; kv_range_table_names.emplace_back(kv_range_table_name); FlushData(kv_range_table_names, callback_data, &SyncCallback); - callback_data->Wait(); + if (yield_fptr != nullptr && resume_fptr != nullptr) + { + callback_data->Wait(yield_fptr, resume_fptr); + } + else + { + callback_data->Wait(); + } if (callback_data->Result().error_code() != EloqDS::remote::DataStoreError::NO_ERROR) { diff --git a/store_handler/data_store_service_client.h b/store_handler/data_store_service_client.h index 0a75c2eb..536c3b58 100644 --- a/store_handler/data_store_service_client.h +++ b/store_handler/data_store_service_client.h @@ -341,7 +341,10 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler txservice::FillStoreSliceCc *load_slice_req) override; bool UpdateRangeSlices(const std::vector - &update_range_slice_reqs) override; + &update_range_slice_reqs, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) + override; bool UpdateRangeSlices(const txservice::TableName &table_name, uint64_t version, diff --git a/store_handler/rocksdb_handler.cpp b/store_handler/rocksdb_handler.cpp index 7c854e81..44046d65 100644 --- a/store_handler/rocksdb_handler.cpp +++ b/store_handler/rocksdb_handler.cpp @@ -1521,8 +1521,12 @@ bool RocksDBHandler::UpdateRangeSlices( } bool RocksDBHandler::UpdateRangeSlices( - const std::vector &update_range_slice_reqs) + const std::vector &update_range_slice_reqs, + const std::function *yield_fptr, + const std::function *resume_fptr) { + (void)yield_fptr; + (void)resume_fptr; LOG(ERROR) << "RocksDBHandler::UpdateRangeSlices not implemented"; // Not implemented assert(false); diff --git a/store_handler/rocksdb_handler.h b/store_handler/rocksdb_handler.h index 37964d16..8cc67e5e 100644 --- a/store_handler/rocksdb_handler.h +++ b/store_handler/rocksdb_handler.h @@ -415,7 +415,10 @@ class RocksDBHandler : public txservice::store::DataStoreHandler uint64_t range_version) override; bool UpdateRangeSlices(const std::vector - &update_range_slice_reqs) override; + &update_range_slice_reqs, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) + override; bool UpsertRanges(const txservice::TableName &table_name, std::vector range_info, diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index 5e0e4799..694776ba 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -1985,7 +1985,9 @@ class LocalCcShards std::string_view, std::vector>> &flush_task_entries, - DataSyncTask::CkptErrorCode ckpt_err); + DataSyncTask::CkptErrorCode ckpt_err, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr); const uint32_t node_id_; // Native node group @@ -2481,7 +2483,10 @@ class LocalCcShards const TxKey *end_key, bool flush_res); - bool UpdateStoreSlices(std::vector &task); + bool UpdateStoreSlices( + std::vector &task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr); bool GetNextRangePartitionId(const TableName &tablename, const TableSchema *table_schema, diff --git a/tx_service/include/store/data_store_handler.h b/tx_service/include/store/data_store_handler.h index ee6ee9f9..5c5a6a7d 100644 --- a/tx_service/include/store/data_store_handler.h +++ b/tx_service/include/store/data_store_handler.h @@ -234,8 +234,12 @@ class DataStoreHandler } virtual bool UpdateRangeSlices( - const std::vector &update_range_slice_reqs) + const std::vector &update_range_slice_reqs, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) { + (void)yield_fptr; + (void)resume_fptr; return false; } diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index e427c067..3e0dfc81 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -3163,7 +3163,9 @@ void LocalCcShards::PostProcessFlushTaskEntries( std::unordered_map>> &flush_task_entries, - DataSyncTask::CkptErrorCode ckpt_err) + DataSyncTask::CkptErrorCode ckpt_err, + const std::function *yield_fptr, + const std::function *resume_fptr) { assert(ckpt_err != DataSyncTask::CkptErrorCode::SCAN_ERROR); @@ -3322,7 +3324,8 @@ void LocalCcShards::PostProcessFlushTaskEntries( if (!pending_update_entries.empty()) { - bool success = UpdateStoreSlices(pending_update_entries); + bool success = + UpdateStoreSlices(pending_update_entries, yield_fptr, resume_fptr); for (auto &entry : pending_update_entries) { @@ -6020,7 +6023,8 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, << " new_usage: " << old_usage - cur_work->pending_flush_size_ << " quota: " << data_sync_mem_controller_.FlushMemoryQuota(); - PostProcessFlushTaskEntries(flush_task_entries, ckpt_err); + PostProcessFlushTaskEntries( + flush_task_entries, ckpt_err, &yield_fn, &resume_fn); } void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk, @@ -6297,7 +6301,9 @@ void LocalCcShards::RangeSplitWorker() } bool LocalCcShards::UpdateStoreSlices( - std::vector &flush_tasks) + std::vector &flush_tasks, + const std::function *yield_fptr, + const std::function *resume_fptr) { std::vector update_range_slice_reqs; @@ -6340,7 +6346,8 @@ bool LocalCcShards::UpdateStoreSlices( if (!update_range_slice_reqs.empty()) { - bool success = store_hd_->UpdateRangeSlices(update_range_slice_reqs); + bool success = store_hd_->UpdateRangeSlices( + update_range_slice_reqs, yield_fptr, resume_fptr); return success; } From d95eb7052cc05f1d0f1d220195700aad4f09f1f6 Mon Sep 17 00:00:00 2001 From: lokax Date: Fri, 27 Feb 2026 11:36:55 +0800 Subject: [PATCH 11/38] 1 worker and fix build --- store_handler/data_store_service_client.cpp | 18 +++++--- .../data_store_service_client_closure.cpp | 20 ++++++++- .../data_store_service_client_closure.h | 35 ++++++++------- tx_service/include/cc/cc_req_misc.h | 24 ++++++++--- tx_service/include/cc/local_cc_shards.h | 20 +++++---- tx_service/src/cc/local_cc_shards.cpp | 43 +++++++++++++------ 6 files changed, 109 insertions(+), 51 deletions(-) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index 1b9c6717..b873a72b 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -445,7 +446,8 @@ bool DataStoreServiceClient::PutAllImpl( // Set up global coordinator sync_putall->total_partitions_ = sync_putall->partition_states_.size(); - // Set coroutine callbacks BEFORE starting async work (see plan risk analysis) + // Set coroutine callbacks BEFORE starting async work (see plan risk + // analysis) if (yield_fptr != nullptr && resume_fptr != nullptr) { sync_putall->SetCoroCallbacks(yield_fptr, resume_fptr); @@ -487,7 +489,9 @@ bool DataStoreServiceClient::PutAllImpl( // Wait for all partitions to complete if (yield_fptr != nullptr && resume_fptr != nullptr) { + LOG(INFO) << "PutAllImpl: Before Wait, this = " << sync_putall; sync_putall->Wait(yield_fptr, resume_fptr); + LOG(INFO) << "PutAllImpl: After Wait, this = " << sync_putall; } else { @@ -1865,7 +1869,7 @@ bool DataStoreServiceClient::UpdateRangeSlices( if (yield_fptr != nullptr && resume_fptr != nullptr) { flush_slices_callback_data->SetCoroCallbacks(yield_fptr, - resume_fptr); + resume_fptr); } std::vector kv_slices_table_names; kv_slices_table_names.emplace_back(kv_range_slices_table_name); @@ -3183,12 +3187,14 @@ bool DataStoreServiceClient::CopyBaseToArchiveImpl( bthread::ConditionVariable cv; size_t flying_cnt = 0; int error_code = 0; - std::vector callback_datas; - callback_datas.reserve(base_vec.size()); + // Use deque: ReadBaseForArchiveCallbackData contains std::atomic + // which is non-copyable/non-movable; vector requires element + // move/copy on realloc, deque does not. + std::deque callback_datas; for (size_t i = 0; i < base_vec.size(); ++i) { - callback_datas.emplace_back(mtx, cv, flying_cnt, error_code, - resume_fptr); + callback_datas.emplace_back( + mtx, cv, flying_cnt, error_code, resume_fptr); } for (size_t base_idx = 0; base_idx < base_vec.size(); ++base_idx) diff --git a/store_handler/data_store_service_client_closure.cpp b/store_handler/data_store_service_client_closure.cpp index cdb747da..66bc141c 100644 --- a/store_handler/data_store_service_client_closure.cpp +++ b/store_handler/data_store_service_client_closure.cpp @@ -523,7 +523,7 @@ void SyncPutAllData::Wait() } void SyncPutAllData::Wait(const std::function *yield_fn, - const std::function *resume_fn) + const std::function *resume_fn) { if (yield_fn == nullptr || resume_fn == nullptr) { @@ -535,7 +535,9 @@ void SyncPutAllData::Wait(const std::function *yield_fn, { waiting_.store(true, std::memory_order_release); lk.unlock(); + LOG(INFO) << "SyncPutAllData Wait: Before yield, this = " << this; (*yield_fn)(); + LOG(INFO) << "SynPutAllData Wait: After yield, this = " << this; lk.lock(); waiting_.store(false, std::memory_order_release); } @@ -550,7 +552,15 @@ void SyncConcurrentRequest::WaitForCapacityAndIncrement() { waiting_.store(true, std::memory_order_release); lk.unlock(); + LOG(INFO) << "SyncConcurrentRequest WaitForCapacityAndIncrement: " + "Before yield, this = " + << this; + (*yield_fn_)(); + + LOG(INFO) << "SyncConcurrentRequest WaitForCapacityAndIncrement: " + "After yield, this = " + << this; lk.lock(); waiting_.store(false, std::memory_order_release); } @@ -572,7 +582,15 @@ void SyncConcurrentRequest::WaitForAll() { waiting_.store(true, std::memory_order_release); lk.unlock(); + LOG(INFO) + << "SyncConcurrentRequest WaitForAll: Before yield, this = " + << this; + (*yield_fn_)(); + + LOG(INFO) + << "SyncConcurrentRequest WaitForAll: After yield, this = " + << this; lk.lock(); waiting_.store(false, std::memory_order_release); } diff --git a/store_handler/data_store_service_client_closure.h b/store_handler/data_store_service_client_closure.h index 9df82f30..89449293 100644 --- a/store_handler/data_store_service_client_closure.h +++ b/store_handler/data_store_service_client_closure.h @@ -109,14 +109,14 @@ struct SyncCallbackData : public Poolable { std::unique_lock lk(mtx_); finished_ = true; - if (resume_fn_ != nullptr && - waiting_.load(std::memory_order_acquire)) + if (resume_fn_ != nullptr && waiting_.load(std::memory_order_acquire)) { waiting_.store(false, std::memory_order_release); + auto *fn = resume_fn_; lk.unlock(); - (*resume_fn_)(); + (*fn)(); } - else + else if (resume_fn_ == nullptr) { cv_.notify_one(); } @@ -305,8 +305,9 @@ struct SyncPutAllData : public Poolable if (resume_fn_ && waiting_.load(std::memory_order_acquire)) { waiting_.store(false, std::memory_order_release); + auto *fn = resume_fn_; lk.unlock(); - (*resume_fn_)(); + (*fn)(); // Do not re-lock: resume_fn may acquire flush_worker_mux; // coroutine will need mux_ when it resumes. Unlock before // resume_fn avoids deadlock. @@ -401,8 +402,9 @@ struct SyncConcurrentRequest : public Poolable if (resume_fn_ && waiting_.load(std::memory_order_acquire)) { waiting_.store(false, std::memory_order_release); + auto *fn = resume_fn_; lk.unlock(); - (*resume_fn_)(); + (*fn)(); // Do not re-lock: resume_fn may acquire flush_worker_mux } else if (!resume_fn_) @@ -564,12 +566,12 @@ void SyncCallback(void *data, struct ReadBaseForArchiveCallbackData { - ReadBaseForArchiveCallbackData(bthread::Mutex &mtx, - bthread::ConditionVariable &cv, - size_t &flying_read_cnt, - int &error_code, - const std::function *resume_fn = - nullptr) + ReadBaseForArchiveCallbackData( + bthread::Mutex &mtx, + bthread::ConditionVariable &cv, + size_t &flying_read_cnt, + int &error_code, + const std::function *resume_fn = nullptr) : mtx_(mtx), cv_(cv), flying_read_cnt_(flying_read_cnt), @@ -646,11 +648,14 @@ struct ReadBaseForArchiveCallbackData if (resume_fn_ && waiting_.load(std::memory_order_acquire)) { waiting_.store(false, std::memory_order_release); + auto *fn = resume_fn_; lk.unlock(); - (*resume_fn_)(); - // Do not re-lock: resume_fn may acquire flush_worker_mux + (*fn)(); + // Do not re-lock: resume_fn may acquire flush_worker_mux. + // Do not access members after fn(): object may be destroyed. + return 0; } - else + else if (!resume_fn_) { cv_.notify_one(); } diff --git a/tx_service/include/cc/cc_req_misc.h b/tx_service/include/cc/cc_req_misc.h index 934efde5..5fa4366d 100644 --- a/tx_service/include/cc/cc_req_misc.h +++ b/tx_service/include/cc/cc_req_misc.h @@ -27,14 +27,15 @@ #include #include #include -#include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -906,7 +907,9 @@ struct WaitableCc : public RunOnTxProcessorCc { waiting_.store(true, std::memory_order_release); lk.unlock(); + LOG(INFO) << "WaitableCc Wait: Before yield"; (*yield_fn)(); + LOG(INFO) << "WaitableCc Wait: After Yield"; lk.lock(); waiting_.store(false, std::memory_order_release); } @@ -941,10 +944,11 @@ struct WaitableCc : public RunOnTxProcessorCc waiting_.load(std::memory_order_acquire)) { waiting_.store(false, std::memory_order_release); + auto *fn = resume_fn_; lk.unlock(); - (*resume_fn_)(); + (*fn)(); } - else + else if (resume_fn_ == nullptr) { cv_.notify_one(); } @@ -963,10 +967,12 @@ struct WaitableCc : public RunOnTxProcessorCc waiting_.load(std::memory_order_acquire)) { waiting_.store(false, std::memory_order_release); + auto *fn = resume_fn_; lk.unlock(); - (*resume_fn_)(); + LOG(INFO) << "WaitableCc Execute: before resume"; + (*fn)(); } - else + else if (resume_fn_ == nullptr) { cv_.notify_one(); } @@ -1059,10 +1065,12 @@ struct UpdateCceCkptTsCc : public CcRequestBase waiting_.load(std::memory_order_acquire)) { waiting_.store(false, std::memory_order_release); + auto *fn = resume_fn_; lk.unlock(); - (*resume_fn_)(); + LOG(INFO) << "UpdateCceCkptTsCc SetFinished: before resume"; + (*fn)(); } - else + else if (resume_fn_ == nullptr) { cv_.notify_one(); } @@ -1091,7 +1099,9 @@ struct UpdateCceCkptTsCc : public CcRequestBase { waiting_.store(true, std::memory_order_release); lk.unlock(); + LOG(INFO) << "UpdateCceCkptTsCc Wait: Before yield"; (*yield_fn)(); + LOG(INFO) << "UpdateCceCkptTsCc Wait: After Yield"; lk.lock(); waiting_.store(false, std::memory_order_release); } diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index 694776ba..d17b3b02 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -90,11 +90,16 @@ struct FlushDataTask; struct CoroCtx { + ~CoroCtx() + { + LOG(INFO) << "CoroCtx destructor, this = " << this; + } + boost::context::continuation coro_; std::unique_ptr task_; - std::function sync_yield_func; // 同步 yield：入队后让出 - std::function yield_fn; // 异步 yield - std::function resume_fn; // 用 weak_ptr 捕获 ctx，无参调用 + std::function sync_yield_func; + std::function yield_fn; + std::function resume_fn; }; struct DataMigrationStatus @@ -2483,10 +2488,9 @@ class LocalCcShards const TxKey *end_key, bool flush_res); - bool UpdateStoreSlices( - std::vector &task, - const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr); + bool UpdateStoreSlices(std::vector &task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr); bool GetNextRangePartitionId(const TableName &tablename, const TableSchema *table_schema, @@ -2531,7 +2535,7 @@ class LocalCcShards 512 * 1024}; // 512KB, guard page for stack overflow detection #else boost::context::pooled_fixedsize_stack flush_coro_stack_allocator_{ - 1024 * 1024}; // 512KB for FlushData call chain + 1024 * 1024}; // 1MB for FlushData call chain #endif void FlushDataWorker(size_t worker_idx); diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 3e0dfc81..282458c1 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -132,13 +132,9 @@ LocalCcShards::LocalCcShards( data_sync_worker_ctx_(conf.at("core_num")), #ifdef EXT_TX_PROC_ENABLED - flush_data_worker_ctx_( - conf.at("core_num") >= 2 - ? std::min(conf.at("core_num") / 2, (uint32_t) 10) - : 1), + flush_data_worker_ctx_(1), #else - flush_data_worker_ctx_( - std::min(static_cast(conf.at("core_num")), 10)), + flush_data_worker_ctx_(1), #endif // cur_flush_buffers_ will be resized in StartBackgroudWorkers() // when worker_num_ is determined @@ -5933,6 +5929,9 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, } } + LOG(INFO) << "FlushDataImpl: start update cce ts, task addr = " << cur_work + << ", worker idx = " << worker_idx; + std::unordered_set updated_ckpt_ts_core_ids; // Update cce ckpt ts in memory if (succ) @@ -5946,6 +5945,8 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, continue; } + auto start_time = std::chrono::steady_clock::now(); + absl::flat_hash_map> cce_entries_map; @@ -5977,6 +5978,14 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, } } + auto stop_time = std::chrono::steady_clock::now(); + auto duration = + std::chrono::duration_cast( + stop_time - start_time); + LOG(INFO) << "FlushDataImpl: UpdateCceCkptTsCc duration = " + << duration.count() << "us" + << ", rec size = " << entry->data_sync_vec_->size(); + if (cce_entries_map.size() > 0) { UpdateCceCkptTsCc update_cce_req( @@ -5996,6 +6005,8 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, } } + LOG(INFO) << "FlushDataImpl: start reset cc, task addr = " << cur_work + << ", worker idx = " << worker_idx; // Notify cc shards that dirty data has been flushed. This will re-enqueue // kickout data cc reqs if there are any. WaitableCc reset_cc( @@ -6070,20 +6081,18 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) std::unique_lock flush_worker_lk(flush_data_worker_ctx_.mux_); while (flush_data_worker_ctx_.status_ == WorkerStatus::Active) { - // 1. 从 resume_queue 获取被唤醒的协程 if (!resume_queue.empty()) { std::shared_ptr ctx = std::move(resume_queue.front()); resume_queue.pop_front(); flush_worker_lk.unlock(); - ctx->coro_ = ctx->coro_.resume(); // 单次 resume，无循环 + ctx->coro_ = ctx->coro_.resume(); flush_worker_lk.lock(); continue; } - // 2. 等待新任务或 resume_queue 通知 flush_data_worker_ctx_.cv_.wait_for( flush_worker_lk, 10s, @@ -6150,7 +6159,6 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) continue; } - // 3. 取任务，以协程执行 FlushData std::unique_ptr cur_work = std::move(pending_flush_work.front()); pending_flush_work.pop_front(); @@ -6164,7 +6172,11 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) flush_coro_stack_allocator_, [this, ctx, worker_idx](continuation &&sink) { - ctx->yield_fn = [&sink]() { sink = sink.resume(); }; + ctx->yield_fn = [&sink]() + { + LOG(INFO) << "CoroCtx yield_fn"; + sink = sink.resume(); + }; ctx->sync_yield_func = [&sink, weak_ctx = std::weak_ptr(ctx), this, @@ -6186,11 +6198,17 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) { if (auto c = weak_ctx.lock()) { + LOG(INFO) + << "CoroCtx resume_fn: coro ctx = " << c.get(); std::lock_guard lk( flush_data_worker_ctx_.mux_); resume_queue_[worker_idx].push_back(std::move(c)); flush_data_worker_ctx_.cv_.notify_all(); } + else + { + LOG(FATAL) << "CoroCtx resume_fn: weak_ctx is expired"; + } }; FlushDataImpl(ctx->task_.get(), worker_idx, @@ -6203,10 +6221,8 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) flush_worker_lk.lock(); } - // Terminate 时清空队列，使用协程方式执行 while (!resume_queue.empty() || !pending_flush_work.empty()) { - // 1. 优先处理 resume_queue 中的协程 if (!resume_queue.empty()) { std::shared_ptr ctx = std::move(resume_queue.front()); @@ -6217,7 +6233,6 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) continue; } - // 2. pending_flush_work 中有任务，以协程方式启动 std::unique_ptr cur_work = std::move(pending_flush_work.front()); pending_flush_work.pop_front(); From ae2497d3d8d028ea8fd8566901947489cb0fbe17 Mon Sep 17 00:00:00 2001 From: lokax Date: Fri, 27 Feb 2026 18:14:24 +0800 Subject: [PATCH 12/38] yield to --- store_handler/data_store_service_client.cpp | 22 ++++++-- store_handler/data_store_service_client.h | 9 +++- store_handler/rocksdb_handler.cpp | 6 ++- store_handler/rocksdb_handler.h | 5 +- tx_service/include/cc/cc_req_misc.h | 8 ++- tx_service/include/cc/local_cc_shards.h | 3 +- tx_service/include/store/data_store_handler.h | 8 +-- tx_service/include/store/int_mem_store.h | 7 ++- tx_service/src/cc/local_cc_shards.cpp | 53 ++++++++++++++++++- 9 files changed, 105 insertions(+), 16 deletions(-) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index b873a72b..7cd3167a 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -316,9 +316,12 @@ bool DataStoreServiceClient::PutAll( std::vector>> &flush_task, const std::function *yield_fptr, - const std::function *resume_fptr) + const std::function *resume_fptr, + const std::function *sync_yield_fptr, + const std::function *has_other_work_fptr) { - return PutAllImpl(flush_task, yield_fptr, resume_fptr); + return PutAllImpl(flush_task, yield_fptr, resume_fptr, sync_yield_fptr, + has_other_work_fptr); } bool DataStoreServiceClient::PutAllImpl( @@ -326,7 +329,9 @@ bool DataStoreServiceClient::PutAllImpl( std::vector>> &flush_task, const std::function *yield_fptr, - const std::function *resume_fptr) + const std::function *resume_fptr, + const std::function *sync_yield_fptr, + const std::function *has_other_work_fptr) { DLOG(INFO) << "DataStoreServiceClient::PutAll called with " << flush_task.size() << " tables to flush."; @@ -454,6 +459,9 @@ bool DataStoreServiceClient::PutAllImpl( } // Start concurrent processing for each partition + constexpr size_t MAX_BATCH_WRITES_WITHOUT_YIELD = 10; + size_t batch_writes_since_yield = 0; + for (size_t i = 0; i < callback_data_list.size(); ++i) { auto *partition_state = sync_putall->partition_states_[i]; @@ -478,6 +486,14 @@ bool DataStoreServiceClient::PutAllImpl( PartitionBatchCallback, first_batch.parts_cnt_per_key, first_batch.parts_cnt_per_record); + batch_writes_since_yield++; + if (sync_yield_fptr != nullptr && has_other_work_fptr != nullptr && + batch_writes_since_yield >= MAX_BATCH_WRITES_WITHOUT_YIELD && + (*has_other_work_fptr)()) + { + (*sync_yield_fptr)(); + batch_writes_since_yield = 0; + } } else { diff --git a/store_handler/data_store_service_client.h b/store_handler/data_store_service_client.h index 536c3b58..95edf53d 100644 --- a/store_handler/data_store_service_client.h +++ b/store_handler/data_store_service_client.h @@ -234,7 +234,10 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler std::vector>> &flush_task, const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr) override; + const std::function *resume_fptr = nullptr, + const std::function *sync_yield_fptr = nullptr, + const std::function *has_other_work_fptr = nullptr) + override; bool NeedPersistKV() override { @@ -624,7 +627,9 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler std::vector>> &flush_task, const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr); + const std::function *resume_fptr = nullptr, + const std::function *sync_yield_fptr = nullptr, + const std::function *has_other_work_fptr = nullptr); bool CopyBaseToArchiveImpl(std::unordered_map< std::string_view, diff --git a/store_handler/rocksdb_handler.cpp b/store_handler/rocksdb_handler.cpp index 44046d65..6119a56f 100644 --- a/store_handler/rocksdb_handler.cpp +++ b/store_handler/rocksdb_handler.cpp @@ -435,10 +435,14 @@ bool RocksDBHandler::PutAll( std::vector>> &batch, const std::function *yield_fptr, - const std::function *resume_fptr) + const std::function *resume_fptr, + const std::function *sync_yield_fptr, + const std::function *has_other_work_fptr) { (void)yield_fptr; (void)resume_fptr; + (void)sync_yield_fptr; + (void)has_other_work_fptr; std::thread::id this_id = std::this_thread::get_id(); if (batch.empty()) { diff --git a/store_handler/rocksdb_handler.h b/store_handler/rocksdb_handler.h index 8cc67e5e..742221c5 100644 --- a/store_handler/rocksdb_handler.h +++ b/store_handler/rocksdb_handler.h @@ -276,7 +276,10 @@ class RocksDBHandler : public txservice::store::DataStoreHandler std::string_view, std::vector>> &batch, const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr) override; + const std::function *resume_fptr = nullptr, + const std::function *sync_yield_fptr = nullptr, + const std::function *has_other_work_fptr = nullptr) + override; /** * @brief indicate end of flush entries in a single ckpt for \@param diff --git a/tx_service/include/cc/cc_req_misc.h b/tx_service/include/cc/cc_req_misc.h index 5fa4366d..0f2f01a5 100644 --- a/tx_service/include/cc/cc_req_misc.h +++ b/tx_service/include/cc/cc_req_misc.h @@ -1113,6 +1113,12 @@ struct UpdateCceCkptTsCc : public CcRequestBase return cce_entries_; } + bool IsFinished() const + { + std::lock_guard lk(mux_); + return unfinished_core_cnt_ == 0; + } + private: absl::flat_hash_map> &cce_entries_; // key: core_idx, value: entry_index @@ -1122,7 +1128,7 @@ struct UpdateCceCkptTsCc : public CcRequestBase NodeGroupId node_group_id_; int64_t term_; TableName table_name_; - bthread::Mutex mux_; + mutable bthread::Mutex mux_; bthread::ConditionVariable cv_; // Coroutine yield/resume support diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index d17b3b02..1a75f73c 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -2542,7 +2542,8 @@ class LocalCcShards void FlushData(std::unique_lock &flush_worker_lk, size_t worker_idx); - bool ShouldYieldFlushData(size_t worker_idx) const; + bool ShouldYieldFlushData(size_t worker_idx); + bool HasOtherWorkToDo(size_t worker_idx); void FlushDataImpl(FlushDataTask *cur_work, size_t worker_idx, const std::function &sync_yield_func, diff --git a/tx_service/include/store/data_store_handler.h b/tx_service/include/store/data_store_handler.h index 5c5a6a7d..0cf23a72 100644 --- a/tx_service/include/store/data_store_handler.h +++ b/tx_service/include/store/data_store_handler.h @@ -104,8 +104,8 @@ class DataStoreHandler const std::function *yield_fptr = nullptr, const std::function *resume_fptr = nullptr) { - (void)yield_fptr; - (void)resume_fptr; + (void) yield_fptr; + (void) resume_fptr; return true; } @@ -238,8 +238,8 @@ class DataStoreHandler const std::function *yield_fptr = nullptr, const std::function *resume_fptr = nullptr) { - (void)yield_fptr; - (void)resume_fptr; + (void) yield_fptr; + (void) resume_fptr; return false; } diff --git a/tx_service/include/store/int_mem_store.h b/tx_service/include/store/int_mem_store.h index ae854959..b0e6dd94 100644 --- a/tx_service/include/store/int_mem_store.h +++ b/tx_service/include/store/int_mem_store.h @@ -59,10 +59,15 @@ class IntMemoryStore : public DataStoreHandler std::vector>> &flush_task, const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr) override + const std::function *resume_fptr = nullptr, + const std::function *sync_yield_fptr = nullptr, + const std::function *has_other_work_fptr = nullptr) + override { (void)yield_fptr; (void)resume_fptr; + (void)sync_yield_fptr; + (void)has_other_work_fptr; assert(false); // for (const auto &ref : batch) // { diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 282458c1..6d404641 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -5848,11 +5848,19 @@ void LocalCcShards::FlushCurrentFlushBuffer() } } -bool LocalCcShards::ShouldYieldFlushData(size_t worker_idx) const +bool LocalCcShards::ShouldYieldFlushData(size_t worker_idx) { + std::lock_guard lk(flush_data_worker_ctx_.mux_); return !pending_flush_work_[worker_idx].empty(); } +bool LocalCcShards::HasOtherWorkToDo(size_t worker_idx) +{ + std::lock_guard lk(flush_data_worker_ctx_.mux_); + return !resume_queue_[worker_idx].empty() || + !pending_flush_work_[worker_idx].empty(); +} + void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, size_t worker_idx, const std::function &sync_yield_func, @@ -5875,7 +5883,10 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, if (succ) { - succ = store_hd_->PutAll(flush_task_entries, &yield_fn, &resume_fn); + std::function has_other_work = + [this, worker_idx]() { return HasOtherWorkToDo(worker_idx); }; + succ = store_hd_->PutAll(flush_task_entries, &yield_fn, &resume_fn, + &sync_yield_func, &has_other_work); if (!succ) { LOG(ERROR) << "DataSync PutAll flush to kv " @@ -5936,6 +5947,11 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, // Update cce ckpt ts in memory if (succ) { + size_t iterations_since_yield = 0; + auto last_yield_time = std::chrono::steady_clock::now(); + constexpr size_t MAX_ITERATIONS_WITHOUT_YIELD = 10; + constexpr auto MAX_TIME_WITHOUT_YIELD = std::chrono::milliseconds(5); + for (auto &[kv_table_name, entries] : flush_task_entries) { for (auto &entry : entries) @@ -5988,6 +6004,12 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, if (cce_entries_map.size() > 0) { + iterations_since_yield++; + auto now = std::chrono::steady_clock::now(); + bool should_sync_yield = HasOtherWorkToDo(worker_idx) && + (iterations_since_yield >= MAX_ITERATIONS_WITHOUT_YIELD || + (now - last_yield_time) >= MAX_TIME_WITHOUT_YIELD); + UpdateCceCkptTsCc update_cce_req( entry->data_sync_task_->node_group_id_, entry->data_sync_task_->node_group_term_, @@ -5999,7 +6021,34 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, updated_ckpt_ts_core_ids.insert(core_idx); EnqueueToCcShard(core_idx, &update_cce_req); } + if (should_sync_yield) + { + sync_yield_func(); + iterations_since_yield = 0; + last_yield_time = std::chrono::steady_clock::now(); + } + + bool was_finished_before_wait = update_cce_req.IsFinished(); update_cce_req.Wait(&yield_fn, &resume_fn); + if (!was_finished_before_wait) + { + iterations_since_yield = 0; + last_yield_time = std::chrono::steady_clock::now(); + } + else + { + now = std::chrono::steady_clock::now(); + bool should_sync_yield_after_wait = + HasOtherWorkToDo(worker_idx) && + (iterations_since_yield >= MAX_ITERATIONS_WITHOUT_YIELD || + (now - last_yield_time) >= MAX_TIME_WITHOUT_YIELD); + if (should_sync_yield_after_wait) + { + sync_yield_func(); + iterations_since_yield = 0; + last_yield_time = std::chrono::steady_clock::now(); + } + } } } } From 983bccf3152e94cda43c3fe634b8e33f26a68822 Mon Sep 17 00:00:00 2001 From: lokax Date: Sat, 28 Feb 2026 14:10:20 +0800 Subject: [PATCH 13/38] yield queue --- tx_service/include/cc/local_cc_shards.h | 6 +- tx_service/src/cc/local_cc_shards.cpp | 277 ++++++++++++++---------- 2 files changed, 166 insertions(+), 117 deletions(-) diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index 1a75f73c..f64c1c93 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -2526,9 +2526,11 @@ class LocalCcShards // Per-worker flush task queues. Each FlushDataWorker processes its // corresponding queue. std::vector>> pending_flush_work_; - // Per-worker queues of coroutine contexts ready to resume (yielded by - // sync_yield_func or resume_fn). + // Per-worker queues of coroutine contexts ready to resume. + // resume_queue_: async resume (resume_fn, e.g. PutAll Wait done). + // sync_yield_queue_: voluntary sync yield (sync_yield_func), lower priority. std::vector>> resume_queue_; + std::vector>> sync_yield_queue_; #ifndef NDEBUG boost::context::protected_fixedsize_stack flush_coro_stack_allocator_{ diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 6d404641..527c6fe0 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -302,6 +302,7 @@ void LocalCcShards::StartBackgroudWorkers() } pending_flush_work_.resize(worker_num); resume_queue_.resize(worker_num); + sync_yield_queue_.resize(worker_num); // Starts flush worker threads firstly. for (int id = 0; id < flush_data_worker_ctx_.worker_num_; id++) @@ -5858,7 +5859,8 @@ bool LocalCcShards::HasOtherWorkToDo(size_t worker_idx) { std::lock_guard lk(flush_data_worker_ctx_.mux_); return !resume_queue_[worker_idx].empty() || - !pending_flush_work_[worker_idx].empty(); + !pending_flush_work_[worker_idx].empty() || + !sync_yield_queue_[worker_idx].empty(); } void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, @@ -5883,10 +5885,13 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, if (succ) { - std::function has_other_work = - [this, worker_idx]() { return HasOtherWorkToDo(worker_idx); }; - succ = store_hd_->PutAll(flush_task_entries, &yield_fn, &resume_fn, - &sync_yield_func, &has_other_work); + std::function has_other_work = [this, worker_idx]() + { return HasOtherWorkToDo(worker_idx); }; + succ = store_hd_->PutAll(flush_task_entries, + &yield_fn, + &resume_fn, + &sync_yield_func, + &has_other_work); if (!succ) { LOG(ERROR) << "DataSync PutAll flush to kv " @@ -6006,8 +6011,10 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, { iterations_since_yield++; auto now = std::chrono::steady_clock::now(); - bool should_sync_yield = HasOtherWorkToDo(worker_idx) && - (iterations_since_yield >= MAX_ITERATIONS_WITHOUT_YIELD || + bool should_sync_yield = + HasOtherWorkToDo(worker_idx) && + (iterations_since_yield >= + MAX_ITERATIONS_WITHOUT_YIELD || (now - last_yield_time) >= MAX_TIME_WITHOUT_YIELD); UpdateCceCkptTsCc update_cce_req( @@ -6040,7 +6047,8 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, now = std::chrono::steady_clock::now(); bool should_sync_yield_after_wait = HasOtherWorkToDo(worker_idx) && - (iterations_since_yield >= MAX_ITERATIONS_WITHOUT_YIELD || + (iterations_since_yield >= + MAX_ITERATIONS_WITHOUT_YIELD || (now - last_yield_time) >= MAX_TIME_WITHOUT_YIELD); if (should_sync_yield_after_wait) { @@ -6117,10 +6125,12 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) assert(worker_idx < static_cast(pending_flush_work_.size())); assert(worker_idx < static_cast(cur_flush_buffers_.size())); assert(worker_idx < static_cast(resume_queue_.size())); + assert(worker_idx < static_cast(sync_yield_queue_.size())); auto &pending_flush_work = pending_flush_work_[worker_idx]; auto &cur_flush_buffer = *cur_flush_buffers_[worker_idx]; auto &resume_queue = resume_queue_[worker_idx]; + auto &sync_yield_queue = sync_yield_queue_[worker_idx]; using clock = std::chrono::steady_clock; using continuation = boost::context::continuation; @@ -6142,17 +6152,100 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) continue; } + if (!pending_flush_work.empty()) + { + std::unique_ptr cur_work = + std::move(pending_flush_work.front()); + pending_flush_work.pop_front(); + flush_data_worker_ctx_.cv_.notify_all(); + flush_worker_lk.unlock(); + + auto ctx = std::make_shared(); + ctx->task_ = std::move(cur_work); + ctx->coro_ = boost::context::callcc( + std::allocator_arg, + flush_coro_stack_allocator_, + [this, ctx, worker_idx](continuation &&sink) + { + ctx->yield_fn = [&sink]() + { + LOG(INFO) << "CoroCtx yield_fn"; + sink = sink.resume(); + }; + ctx->sync_yield_func = + [&sink, + weak_ctx = std::weak_ptr(ctx), + this, + worker_idx]() + { + if (auto c = weak_ctx.lock()) + { + { + std::lock_guard lk( + flush_data_worker_ctx_.mux_); + sync_yield_queue_[worker_idx].push_back( + std::move(c)); + flush_data_worker_ctx_.cv_.notify_all(); + } + sink = sink.resume(); + } + }; + ctx->resume_fn = [this, + worker_idx, + weak_ctx = std::weak_ptr(ctx)]() + { + if (auto c = weak_ctx.lock()) + { + LOG(INFO) + << "CoroCtx resume_fn: coro ctx = " << c.get(); + std::lock_guard lk( + flush_data_worker_ctx_.mux_); + resume_queue_[worker_idx].push_back(std::move(c)); + flush_data_worker_ctx_.cv_.notify_all(); + } + else + { + LOG(FATAL) + << "CoroCtx resume_fn: weak_ctx is expired"; + } + }; + FlushDataImpl(ctx->task_.get(), + worker_idx, + ctx->sync_yield_func, + ctx->yield_fn, + ctx->resume_fn); + return std::move(sink); + }); + + flush_worker_lk.lock(); + continue; + } + + if (!sync_yield_queue.empty()) + { + std::shared_ptr ctx = std::move(sync_yield_queue.front()); + sync_yield_queue.pop_front(); + flush_worker_lk.unlock(); + + ctx->coro_ = ctx->coro_.resume(); + + flush_worker_lk.lock(); + continue; + } + flush_data_worker_ctx_.cv_.wait_for( flush_worker_lk, 10s, [this, &pending_flush_work, &resume_queue, + &sync_yield_queue, &cur_flush_buffer, &previous_flush_size, &previous_size_update_time] { if (!pending_flush_work.empty() || !resume_queue.empty() || + !sync_yield_queue.empty() || flush_data_worker_ctx_.status_ == WorkerStatus::Terminated) { return true; @@ -6203,74 +6296,11 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) return false; }); - if (pending_flush_work.empty()) - { - continue; - } - - std::unique_ptr cur_work = - std::move(pending_flush_work.front()); - pending_flush_work.pop_front(); - flush_data_worker_ctx_.cv_.notify_all(); - flush_worker_lk.unlock(); - - auto ctx = std::make_shared(); - ctx->task_ = std::move(cur_work); - ctx->coro_ = boost::context::callcc( - std::allocator_arg, - flush_coro_stack_allocator_, - [this, ctx, worker_idx](continuation &&sink) - { - ctx->yield_fn = [&sink]() - { - LOG(INFO) << "CoroCtx yield_fn"; - sink = sink.resume(); - }; - ctx->sync_yield_func = [&sink, - weak_ctx = std::weak_ptr(ctx), - this, - worker_idx]() - { - if (auto c = weak_ctx.lock()) - { - { - std::lock_guard lk( - flush_data_worker_ctx_.mux_); - resume_queue_[worker_idx].push_back(std::move(c)); - flush_data_worker_ctx_.cv_.notify_all(); - } - sink = sink.resume(); - } - }; - ctx->resume_fn = - [this, worker_idx, weak_ctx = std::weak_ptr(ctx)]() - { - if (auto c = weak_ctx.lock()) - { - LOG(INFO) - << "CoroCtx resume_fn: coro ctx = " << c.get(); - std::lock_guard lk( - flush_data_worker_ctx_.mux_); - resume_queue_[worker_idx].push_back(std::move(c)); - flush_data_worker_ctx_.cv_.notify_all(); - } - else - { - LOG(FATAL) << "CoroCtx resume_fn: weak_ctx is expired"; - } - }; - FlushDataImpl(ctx->task_.get(), - worker_idx, - ctx->sync_yield_func, - ctx->yield_fn, - ctx->resume_fn); - return std::move(sink); - }); - - flush_worker_lk.lock(); + continue; } - while (!resume_queue.empty() || !pending_flush_work.empty()) + while (!resume_queue.empty() || !pending_flush_work.empty() || + !sync_yield_queue.empty()) { if (!resume_queue.empty()) { @@ -6282,56 +6312,73 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) continue; } - std::unique_ptr cur_work = - std::move(pending_flush_work.front()); - pending_flush_work.pop_front(); - flush_data_worker_ctx_.cv_.notify_all(); - flush_worker_lk.unlock(); - - auto ctx = std::make_shared(); - ctx->task_ = std::move(cur_work); - ctx->coro_ = boost::context::callcc( - std::allocator_arg, - flush_coro_stack_allocator_, - [this, ctx, worker_idx](continuation &&sink) - { - ctx->yield_fn = [&sink]() { sink = sink.resume(); }; - ctx->sync_yield_func = [&sink, - weak_ctx = std::weak_ptr(ctx), - this, - worker_idx]() + if (!pending_flush_work.empty()) + { + std::unique_ptr cur_work = + std::move(pending_flush_work.front()); + pending_flush_work.pop_front(); + flush_data_worker_ctx_.cv_.notify_all(); + flush_worker_lk.unlock(); + + auto ctx = std::make_shared(); + ctx->task_ = std::move(cur_work); + ctx->coro_ = boost::context::callcc( + std::allocator_arg, + flush_coro_stack_allocator_, + [this, ctx, worker_idx](continuation &&sink) { - if (auto c = weak_ctx.lock()) + ctx->yield_fn = [&sink]() { sink = sink.resume(); }; + ctx->sync_yield_func = + [&sink, + weak_ctx = std::weak_ptr(ctx), + this, + worker_idx]() + { + if (auto c = weak_ctx.lock()) + { + { + std::lock_guard lk( + flush_data_worker_ctx_.mux_); + sync_yield_queue_[worker_idx].push_back( + std::move(c)); + flush_data_worker_ctx_.cv_.notify_all(); + } + sink = sink.resume(); + } + }; + ctx->resume_fn = [this, + worker_idx, + weak_ctx = std::weak_ptr(ctx)]() { + if (auto c = weak_ctx.lock()) { std::lock_guard lk( flush_data_worker_ctx_.mux_); resume_queue_[worker_idx].push_back(std::move(c)); flush_data_worker_ctx_.cv_.notify_all(); } - sink = sink.resume(); - } - }; - ctx->resume_fn = - [this, worker_idx, weak_ctx = std::weak_ptr(ctx)]() - { - if (auto c = weak_ctx.lock()) - { - std::lock_guard lk( - flush_data_worker_ctx_.mux_); - resume_queue_[worker_idx].push_back(std::move(c)); - flush_data_worker_ctx_.cv_.notify_all(); - } - }; - FlushDataImpl(ctx->task_.get(), - worker_idx, - ctx->sync_yield_func, - ctx->yield_fn, - ctx->resume_fn); - return std::move(sink); - }); + }; + FlushDataImpl(ctx->task_.get(), + worker_idx, + ctx->sync_yield_func, + ctx->yield_fn, + ctx->resume_fn); + return std::move(sink); + }); + + flush_worker_lk.lock(); + continue; + } - flush_worker_lk.lock(); + if (!sync_yield_queue.empty()) + { + std::shared_ptr ctx = std::move(sync_yield_queue.front()); + sync_yield_queue.pop_front(); + flush_worker_lk.unlock(); + ctx->coro_ = ctx->coro_.resume(); + flush_worker_lk.lock(); + continue; + } } } From 6fcb60b48b7ddb8ca7fbb168104ad6c6d3da9a82 Mon Sep 17 00:00:00 2001 From: lokax Date: Sat, 28 Feb 2026 14:14:35 +0800 Subject: [PATCH 14/38] remove yield queue --- tx_service/include/cc/local_cc_shards.h | 6 +-- tx_service/src/cc/local_cc_shards.cpp | 49 +++++-------------------- 2 files changed, 12 insertions(+), 43 deletions(-) diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index f64c1c93..edab8ff0 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -2526,11 +2526,9 @@ class LocalCcShards // Per-worker flush task queues. Each FlushDataWorker processes its // corresponding queue. std::vector>> pending_flush_work_; - // Per-worker queues of coroutine contexts ready to resume. - // resume_queue_: async resume (resume_fn, e.g. PutAll Wait done). - // sync_yield_queue_: voluntary sync yield (sync_yield_func), lower priority. + // Per-worker queues of coroutine contexts ready to resume (resume_fn or + // sync_yield_func). std::vector>> resume_queue_; - std::vector>> sync_yield_queue_; #ifndef NDEBUG boost::context::protected_fixedsize_stack flush_coro_stack_allocator_{ diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 527c6fe0..ec7d50f6 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -302,7 +302,6 @@ void LocalCcShards::StartBackgroudWorkers() } pending_flush_work_.resize(worker_num); resume_queue_.resize(worker_num); - sync_yield_queue_.resize(worker_num); // Starts flush worker threads firstly. for (int id = 0; id < flush_data_worker_ctx_.worker_num_; id++) @@ -5859,8 +5858,7 @@ bool LocalCcShards::HasOtherWorkToDo(size_t worker_idx) { std::lock_guard lk(flush_data_worker_ctx_.mux_); return !resume_queue_[worker_idx].empty() || - !pending_flush_work_[worker_idx].empty() || - !sync_yield_queue_[worker_idx].empty(); + !pending_flush_work_[worker_idx].empty(); } void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, @@ -6125,12 +6123,10 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) assert(worker_idx < static_cast(pending_flush_work_.size())); assert(worker_idx < static_cast(cur_flush_buffers_.size())); assert(worker_idx < static_cast(resume_queue_.size())); - assert(worker_idx < static_cast(sync_yield_queue_.size())); auto &pending_flush_work = pending_flush_work_[worker_idx]; auto &cur_flush_buffer = *cur_flush_buffers_[worker_idx]; auto &resume_queue = resume_queue_[worker_idx]; - auto &sync_yield_queue = sync_yield_queue_[worker_idx]; using clock = std::chrono::steady_clock; using continuation = boost::context::continuation; @@ -6140,18 +6136,6 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) std::unique_lock flush_worker_lk(flush_data_worker_ctx_.mux_); while (flush_data_worker_ctx_.status_ == WorkerStatus::Active) { - if (!resume_queue.empty()) - { - std::shared_ptr ctx = std::move(resume_queue.front()); - resume_queue.pop_front(); - flush_worker_lk.unlock(); - - ctx->coro_ = ctx->coro_.resume(); - - flush_worker_lk.lock(); - continue; - } - if (!pending_flush_work.empty()) { std::unique_ptr cur_work = @@ -6183,7 +6167,7 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) { std::lock_guard lk( flush_data_worker_ctx_.mux_); - sync_yield_queue_[worker_idx].push_back( + resume_queue_[worker_idx].push_back( std::move(c)); flush_data_worker_ctx_.cv_.notify_all(); } @@ -6221,10 +6205,10 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) continue; } - if (!sync_yield_queue.empty()) + if (!resume_queue.empty()) { - std::shared_ptr ctx = std::move(sync_yield_queue.front()); - sync_yield_queue.pop_front(); + std::shared_ptr ctx = std::move(resume_queue.front()); + resume_queue.pop_front(); flush_worker_lk.unlock(); ctx->coro_ = ctx->coro_.resume(); @@ -6239,13 +6223,11 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) [this, &pending_flush_work, &resume_queue, - &sync_yield_queue, &cur_flush_buffer, &previous_flush_size, &previous_size_update_time] { if (!pending_flush_work.empty() || !resume_queue.empty() || - !sync_yield_queue.empty() || flush_data_worker_ctx_.status_ == WorkerStatus::Terminated) { return true; @@ -6299,19 +6281,8 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) continue; } - while (!resume_queue.empty() || !pending_flush_work.empty() || - !sync_yield_queue.empty()) + while (!resume_queue.empty() || !pending_flush_work.empty()) { - if (!resume_queue.empty()) - { - std::shared_ptr ctx = std::move(resume_queue.front()); - resume_queue.pop_front(); - flush_worker_lk.unlock(); - ctx->coro_ = ctx->coro_.resume(); - flush_worker_lk.lock(); - continue; - } - if (!pending_flush_work.empty()) { std::unique_ptr cur_work = @@ -6339,7 +6310,7 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) { std::lock_guard lk( flush_data_worker_ctx_.mux_); - sync_yield_queue_[worker_idx].push_back( + resume_queue_[worker_idx].push_back( std::move(c)); flush_data_worker_ctx_.cv_.notify_all(); } @@ -6370,10 +6341,10 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) continue; } - if (!sync_yield_queue.empty()) + if (!resume_queue.empty()) { - std::shared_ptr ctx = std::move(sync_yield_queue.front()); - sync_yield_queue.pop_front(); + std::shared_ptr ctx = std::move(resume_queue.front()); + resume_queue.pop_front(); flush_worker_lk.unlock(); ctx->coro_ = ctx->coro_.resume(); flush_worker_lk.lock(); From f01465ff2d899d6abcf960c41447935640206002 Mon Sep 17 00:00:00 2001 From: lokax Date: Sat, 28 Feb 2026 14:45:58 +0800 Subject: [PATCH 15/38] yield --- store_handler/data_store_service_client.cpp | 17 +++--- store_handler/data_store_service_client.h | 6 +- store_handler/rocksdb_handler.cpp | 4 +- store_handler/rocksdb_handler.h | 3 +- tx_service/include/cc/local_cc_shards.h | 1 - tx_service/include/store/data_store_handler.h | 4 +- tx_service/include/store/int_mem_store.h | 4 +- tx_service/src/cc/local_cc_shards.cpp | 61 +++++-------------- 8 files changed, 31 insertions(+), 69 deletions(-) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index 7cd3167a..9b4d2cda 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -317,11 +317,9 @@ bool DataStoreServiceClient::PutAll( &flush_task, const std::function *yield_fptr, const std::function *resume_fptr, - const std::function *sync_yield_fptr, - const std::function *has_other_work_fptr) + const std::function *sync_yield_fptr) { - return PutAllImpl(flush_task, yield_fptr, resume_fptr, sync_yield_fptr, - has_other_work_fptr); + return PutAllImpl(flush_task, yield_fptr, resume_fptr, sync_yield_fptr); } bool DataStoreServiceClient::PutAllImpl( @@ -330,8 +328,7 @@ bool DataStoreServiceClient::PutAllImpl( &flush_task, const std::function *yield_fptr, const std::function *resume_fptr, - const std::function *sync_yield_fptr, - const std::function *has_other_work_fptr) + const std::function *sync_yield_fptr) { DLOG(INFO) << "DataStoreServiceClient::PutAll called with " << flush_task.size() << " tables to flush."; @@ -487,10 +484,12 @@ bool DataStoreServiceClient::PutAllImpl( first_batch.parts_cnt_per_key, first_batch.parts_cnt_per_record); batch_writes_since_yield++; - if (sync_yield_fptr != nullptr && has_other_work_fptr != nullptr && - batch_writes_since_yield >= MAX_BATCH_WRITES_WITHOUT_YIELD && - (*has_other_work_fptr)()) + if (sync_yield_fptr != nullptr && + batch_writes_since_yield >= MAX_BATCH_WRITES_WITHOUT_YIELD) { + LOG(INFO) << "PutAllImpl: Before sync yield, this = " + << sync_putall << ", batch_writes_since_yield = " + << batch_writes_since_yield; (*sync_yield_fptr)(); batch_writes_since_yield = 0; } diff --git a/store_handler/data_store_service_client.h b/store_handler/data_store_service_client.h index 95edf53d..955fd320 100644 --- a/store_handler/data_store_service_client.h +++ b/store_handler/data_store_service_client.h @@ -235,8 +235,7 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler &flush_task, const std::function *yield_fptr = nullptr, const std::function *resume_fptr = nullptr, - const std::function *sync_yield_fptr = nullptr, - const std::function *has_other_work_fptr = nullptr) + const std::function *sync_yield_fptr = nullptr) override; bool NeedPersistKV() override @@ -628,8 +627,7 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler &flush_task, const std::function *yield_fptr = nullptr, const std::function *resume_fptr = nullptr, - const std::function *sync_yield_fptr = nullptr, - const std::function *has_other_work_fptr = nullptr); + const std::function *sync_yield_fptr = nullptr); bool CopyBaseToArchiveImpl(std::unordered_map< std::string_view, diff --git a/store_handler/rocksdb_handler.cpp b/store_handler/rocksdb_handler.cpp index 6119a56f..2465d64b 100644 --- a/store_handler/rocksdb_handler.cpp +++ b/store_handler/rocksdb_handler.cpp @@ -436,13 +436,11 @@ bool RocksDBHandler::PutAll( &batch, const std::function *yield_fptr, const std::function *resume_fptr, - const std::function *sync_yield_fptr, - const std::function *has_other_work_fptr) + const std::function *sync_yield_fptr) { (void)yield_fptr; (void)resume_fptr; (void)sync_yield_fptr; - (void)has_other_work_fptr; std::thread::id this_id = std::this_thread::get_id(); if (batch.empty()) { diff --git a/store_handler/rocksdb_handler.h b/store_handler/rocksdb_handler.h index 742221c5..f3615686 100644 --- a/store_handler/rocksdb_handler.h +++ b/store_handler/rocksdb_handler.h @@ -277,8 +277,7 @@ class RocksDBHandler : public txservice::store::DataStoreHandler std::vector>> &batch, const std::function *yield_fptr = nullptr, const std::function *resume_fptr = nullptr, - const std::function *sync_yield_fptr = nullptr, - const std::function *has_other_work_fptr = nullptr) + const std::function *sync_yield_fptr = nullptr) override; /** diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index edab8ff0..2b07c219 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -2543,7 +2543,6 @@ class LocalCcShards size_t worker_idx); bool ShouldYieldFlushData(size_t worker_idx); - bool HasOtherWorkToDo(size_t worker_idx); void FlushDataImpl(FlushDataTask *cur_work, size_t worker_idx, const std::function &sync_yield_func, diff --git a/tx_service/include/store/data_store_handler.h b/tx_service/include/store/data_store_handler.h index 0cf23a72..0aa5fa4d 100644 --- a/tx_service/include/store/data_store_handler.h +++ b/tx_service/include/store/data_store_handler.h @@ -84,13 +84,15 @@ class DataStoreHandler * data_sync_vec_ in each flush task entry. * @return whether all entries are written to data store successfully */ + virtual bool PutAll( std::unordered_map< std::string_view, std::vector>> &flush_task, const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr) = 0; + const std::function *resume_fptr = nullptr, + const std::function *sync_yield_fptr = nullptr) = 0; /** * @brief indicate end of flush entries in a single ckpt for \@param batch diff --git a/tx_service/include/store/int_mem_store.h b/tx_service/include/store/int_mem_store.h index b0e6dd94..3ba32baf 100644 --- a/tx_service/include/store/int_mem_store.h +++ b/tx_service/include/store/int_mem_store.h @@ -60,14 +60,12 @@ class IntMemoryStore : public DataStoreHandler &flush_task, const std::function *yield_fptr = nullptr, const std::function *resume_fptr = nullptr, - const std::function *sync_yield_fptr = nullptr, - const std::function *has_other_work_fptr = nullptr) + const std::function *sync_yield_fptr = nullptr) override { (void)yield_fptr; (void)resume_fptr; (void)sync_yield_fptr; - (void)has_other_work_fptr; assert(false); // for (const auto &ref : batch) // { diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index ec7d50f6..21de3184 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -5854,13 +5854,6 @@ bool LocalCcShards::ShouldYieldFlushData(size_t worker_idx) return !pending_flush_work_[worker_idx].empty(); } -bool LocalCcShards::HasOtherWorkToDo(size_t worker_idx) -{ - std::lock_guard lk(flush_data_worker_ctx_.mux_); - return !resume_queue_[worker_idx].empty() || - !pending_flush_work_[worker_idx].empty(); -} - void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, size_t worker_idx, const std::function &sync_yield_func, @@ -5883,13 +5876,8 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, if (succ) { - std::function has_other_work = [this, worker_idx]() - { return HasOtherWorkToDo(worker_idx); }; - succ = store_hd_->PutAll(flush_task_entries, - &yield_fn, - &resume_fn, - &sync_yield_func, - &has_other_work); + succ = store_hd_->PutAll( + flush_task_entries, &yield_fn, &resume_fn, &sync_yield_func); if (!succ) { LOG(ERROR) << "DataSync PutAll flush to kv " @@ -5951,9 +5939,7 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, if (succ) { size_t iterations_since_yield = 0; - auto last_yield_time = std::chrono::steady_clock::now(); constexpr size_t MAX_ITERATIONS_WITHOUT_YIELD = 10; - constexpr auto MAX_TIME_WITHOUT_YIELD = std::chrono::milliseconds(5); for (auto &[kv_table_name, entries] : flush_task_entries) { @@ -6007,14 +5993,6 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, if (cce_entries_map.size() > 0) { - iterations_since_yield++; - auto now = std::chrono::steady_clock::now(); - bool should_sync_yield = - HasOtherWorkToDo(worker_idx) && - (iterations_since_yield >= - MAX_ITERATIONS_WITHOUT_YIELD || - (now - last_yield_time) >= MAX_TIME_WITHOUT_YIELD); - UpdateCceCkptTsCc update_cce_req( entry->data_sync_task_->node_group_id_, entry->data_sync_task_->node_group_term_, @@ -6026,35 +6004,24 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, updated_ckpt_ts_core_ids.insert(core_idx); EnqueueToCcShard(core_idx, &update_cce_req); } - if (should_sync_yield) - { - sync_yield_func(); - iterations_since_yield = 0; - last_yield_time = std::chrono::steady_clock::now(); - } - bool was_finished_before_wait = update_cce_req.IsFinished(); - update_cce_req.Wait(&yield_fn, &resume_fn); - if (!was_finished_before_wait) - { - iterations_since_yield = 0; - last_yield_time = std::chrono::steady_clock::now(); - } - else + bool is_finished = update_cce_req.IsFinished(); + if (is_finished) { - now = std::chrono::steady_clock::now(); - bool should_sync_yield_after_wait = - HasOtherWorkToDo(worker_idx) && - (iterations_since_yield >= - MAX_ITERATIONS_WITHOUT_YIELD || - (now - last_yield_time) >= MAX_TIME_WITHOUT_YIELD); - if (should_sync_yield_after_wait) + iterations_since_yield++; + if (iterations_since_yield >= + MAX_ITERATIONS_WITHOUT_YIELD) { + LOG(INFO) + << "FlushDataImpl: Before sync yield, task " + "addr = " + << cur_work << ", worker idx = " << worker_idx; sync_yield_func(); iterations_since_yield = 0; - last_yield_time = std::chrono::steady_clock::now(); } } + + update_cce_req.Wait(&yield_fn, &resume_fn); } } } @@ -6167,6 +6134,8 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) { std::lock_guard lk( flush_data_worker_ctx_.mux_); + LOG(INFO) << "CoroCtx sync_yield_func: push " + "ctx to resume_queue"; resume_queue_[worker_idx].push_back( std::move(c)); flush_data_worker_ctx_.cv_.notify_all(); From 300f1cfc97b0753236d93d489513e6391fcf1eef Mon Sep 17 00:00:00 2001 From: lokax Date: Sat, 28 Feb 2026 14:58:56 +0800 Subject: [PATCH 16/38] debug log --- store_handler/data_store_service_client.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index 9b4d2cda..0c42f6d9 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -1805,6 +1805,8 @@ bool DataStoreServiceClient::UpdateRangeSlices( slice_plans.reserve(update_range_slice_reqs.size()); RangeMetadataAccumulator meta_acc; + auto start_time = std::chrono::steady_clock::now(); + // 1- First pass: Prepare slice batches and accumulate metadata for all // ranges for (auto &req : update_range_slice_reqs) @@ -1844,6 +1846,14 @@ bool DataStoreServiceClient::UpdateRangeSlices( meta_acc); } + auto end_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast( + end_time - start_time); + LOG(INFO) << "UpdateRangeSlices: Prepare slice batches and accumulate " + "metadata for all ranges took " + << duration.count() << "us" + << ", req size = " << update_range_slice_reqs.size(); + // 2- Dispatch slice batches for all ranges concurrently (shared // SyncConcurrentRequest) SyncConcurrentRequest *slice_sync_concurrent = @@ -1863,8 +1873,10 @@ bool DataStoreServiceClient::UpdateRangeSlices( slice_sync_concurrent); } + LOG(INFO) << "UpdateRangeSlices: before WaitAll slice sync"; // 3- Wait for slice requests to complete slice_sync_concurrent->WaitForAll(); + LOG(INFO) << "UpdateRangeSlices: after WaitAll slice sync"; if (slice_sync_concurrent->result_.error_code() != remote::DataStoreError::NO_ERROR) @@ -1919,8 +1931,10 @@ bool DataStoreServiceClient::UpdateRangeSlices( DispatchRangeMetadataBatches( kv_range_table_name, meta_acc, meta_sync_concurrent); + LOG(INFO) << "UpdateRangeSlices: before WaitAll meta sync"; // 5- Wait for metadata requests to complete meta_sync_concurrent->WaitForAll(); + LOG(INFO) << "UpdateRangeSlices: after WaitAll meta sync"; // 6- Check for errors if (meta_sync_concurrent->result_.error_code() != @@ -1946,7 +1960,9 @@ bool DataStoreServiceClient::UpdateRangeSlices( FlushData(kv_range_table_names, callback_data, &SyncCallback); if (yield_fptr != nullptr && resume_fptr != nullptr) { + LOG(INFO) << "UpdateRangeSlices: before Wait meta sync"; callback_data->Wait(yield_fptr, resume_fptr); + LOG(INFO) << "UpdateRangeSlices: after Wait meta sync"; } else { From 3f3b8a15d0d5d4331ca099db4c85c7059950a841 Mon Sep 17 00:00:00 2001 From: lokax Date: Sat, 28 Feb 2026 15:49:36 +0800 Subject: [PATCH 17/38] yield update range slices --- store_handler/data_store_service_client.cpp | 35 +++++++++++++++++-- store_handler/data_store_service_client.h | 3 +- store_handler/rocksdb_handler.cpp | 4 ++- store_handler/rocksdb_handler.h | 3 +- tx_service/include/cc/local_cc_shards.h | 6 ++-- tx_service/include/store/data_store_handler.h | 4 ++- tx_service/src/cc/local_cc_shards.cpp | 29 +++++++++++---- 7 files changed, 69 insertions(+), 15 deletions(-) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index 0c42f6d9..aaeb9b07 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -1794,7 +1794,8 @@ void DataStoreServiceClient::DispatchRangeMetadataBatches( bool DataStoreServiceClient::UpdateRangeSlices( const std::vector &update_range_slice_reqs, const std::function *yield_fptr, - const std::function *resume_fptr) + const std::function *resume_fptr, + const std::function *sync_yield_fptr) { if (update_range_slice_reqs.empty()) { @@ -1807,6 +1808,9 @@ bool DataStoreServiceClient::UpdateRangeSlices( auto start_time = std::chrono::steady_clock::now(); + constexpr size_t MAX_ITERATIONS_WITHOUT_YIELD = 10; + size_t iterations_since_yield = 0; + // 1- First pass: Prepare slice batches and accumulate metadata for all // ranges for (auto &req : update_range_slice_reqs) @@ -1844,6 +1848,18 @@ bool DataStoreServiceClient::UpdateRangeSlices( segment_cnt, range_size, meta_acc); + + if (sync_yield_fptr != nullptr) + { + ++iterations_since_yield; + if (iterations_since_yield >= MAX_ITERATIONS_WITHOUT_YIELD) + { + LOG(INFO) << "UpdateRangeSlices: prepare Before sync yield"; + (*sync_yield_fptr)(); + LOG(INFO) << "UpdateRangeSlices: prepare After sync yield"; + iterations_since_yield = 0; + } + } } auto end_time = std::chrono::steady_clock::now(); @@ -1864,13 +1880,26 @@ bool DataStoreServiceClient::UpdateRangeSlices( { slice_sync_concurrent->SetCoroCallbacks(yield_fptr, resume_fptr); } - for (const auto &[kv_partition_id, slice_plans] : slice_plans) + iterations_since_yield = 0; + for (const auto &[kv_partition_id, plans] : slice_plans) { // Call DispatchRangeSliceBatches once with all plans DispatchRangeSliceBatches(kv_range_slices_table_name, kv_partition_id, - slice_plans, + plans, slice_sync_concurrent); + + if (sync_yield_fptr != nullptr) + { + ++iterations_since_yield; + if (iterations_since_yield >= MAX_ITERATIONS_WITHOUT_YIELD) + { + LOG(INFO) << "UpdateRangeSlices: Before sync yield"; + (*sync_yield_fptr)(); + LOG(INFO) << "UpdateRangeSlices: After sync yield"; + iterations_since_yield = 0; + } + } } LOG(INFO) << "UpdateRangeSlices: before WaitAll slice sync"; diff --git a/store_handler/data_store_service_client.h b/store_handler/data_store_service_client.h index 955fd320..f77425fe 100644 --- a/store_handler/data_store_service_client.h +++ b/store_handler/data_store_service_client.h @@ -345,7 +345,8 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler bool UpdateRangeSlices(const std::vector &update_range_slice_reqs, const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr) + const std::function *resume_fptr = nullptr, + const std::function *sync_yield_fptr = nullptr) override; bool UpdateRangeSlices(const txservice::TableName &table_name, diff --git a/store_handler/rocksdb_handler.cpp b/store_handler/rocksdb_handler.cpp index 2465d64b..981a2a62 100644 --- a/store_handler/rocksdb_handler.cpp +++ b/store_handler/rocksdb_handler.cpp @@ -1525,10 +1525,12 @@ bool RocksDBHandler::UpdateRangeSlices( bool RocksDBHandler::UpdateRangeSlices( const std::vector &update_range_slice_reqs, const std::function *yield_fptr, - const std::function *resume_fptr) + const std::function *resume_fptr, + const std::function *sync_yield_fptr) { (void)yield_fptr; (void)resume_fptr; + (void)sync_yield_fptr; LOG(ERROR) << "RocksDBHandler::UpdateRangeSlices not implemented"; // Not implemented assert(false); diff --git a/store_handler/rocksdb_handler.h b/store_handler/rocksdb_handler.h index f3615686..72eb848d 100644 --- a/store_handler/rocksdb_handler.h +++ b/store_handler/rocksdb_handler.h @@ -419,7 +419,8 @@ class RocksDBHandler : public txservice::store::DataStoreHandler bool UpdateRangeSlices(const std::vector &update_range_slice_reqs, const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr) + const std::function *resume_fptr = nullptr, + const std::function *sync_yield_fptr = nullptr) override; bool UpsertRanges(const txservice::TableName &table_name, diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index 2b07c219..081a5056 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -1992,7 +1992,8 @@ class LocalCcShards &flush_task_entries, DataSyncTask::CkptErrorCode ckpt_err, const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr); + const std::function *resume_fptr = nullptr, + const std::function *sync_yield_fptr = nullptr); const uint32_t node_id_; // Native node group @@ -2490,7 +2491,8 @@ class LocalCcShards bool UpdateStoreSlices(std::vector &task, const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr); + const std::function *resume_fptr = nullptr, + const std::function *sync_yield_fptr = nullptr); bool GetNextRangePartitionId(const TableName &tablename, const TableSchema *table_schema, diff --git a/tx_service/include/store/data_store_handler.h b/tx_service/include/store/data_store_handler.h index 0aa5fa4d..e7f8efb4 100644 --- a/tx_service/include/store/data_store_handler.h +++ b/tx_service/include/store/data_store_handler.h @@ -238,10 +238,12 @@ class DataStoreHandler virtual bool UpdateRangeSlices( const std::vector &update_range_slice_reqs, const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr) + const std::function *resume_fptr = nullptr, + const std::function *sync_yield_fptr = nullptr) { (void) yield_fptr; (void) resume_fptr; + (void) sync_yield_fptr; return false; } diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 21de3184..6f5365d1 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -3161,7 +3161,8 @@ void LocalCcShards::PostProcessFlushTaskEntries( &flush_task_entries, DataSyncTask::CkptErrorCode ckpt_err, const std::function *yield_fptr, - const std::function *resume_fptr) + const std::function *resume_fptr, + const std::function *sync_yield_fptr) { assert(ckpt_err != DataSyncTask::CkptErrorCode::SCAN_ERROR); @@ -3320,8 +3321,8 @@ void LocalCcShards::PostProcessFlushTaskEntries( if (!pending_update_entries.empty()) { - bool success = - UpdateStoreSlices(pending_update_entries, yield_fptr, resume_fptr); + bool success = UpdateStoreSlices( + pending_update_entries, yield_fptr, resume_fptr, sync_yield_fptr); for (auto &entry : pending_update_entries) { @@ -4801,6 +4802,9 @@ void LocalCcShards::DataSyncForHashPartition( data_sync_task->flight_task_cnt_ += 1; } + auto start_scan_time = std::chrono::steady_clock::now(); + size_t debug_data_size = 0; + for (size_t i = 0; i < partition_number_this_core; i += partition_number_per_scan) { @@ -5084,6 +5088,8 @@ void LocalCcShards::DataSyncForHashPartition( } #endif + debug_data_size += flush_data_size; + uint64_t old_usage = data_sync_mem_controller_.AllocateFlushDataMemQuota( flush_data_size); @@ -5284,6 +5290,12 @@ void LocalCcShards::DataSyncForHashPartition( } } + auto stop_scan_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast( + stop_scan_time - start_scan_time); + LOG(INFO) << "FlushDataImpl: scan duration = " << duration.count() << "us" + << ", data size = " << debug_data_size; + PostProcessHashPartitionDataSyncTask(std::move(data_sync_task), data_sync_txm, DataSyncTask::CkptErrorCode::NO_ERROR); @@ -5794,6 +5806,8 @@ void LocalCcShards::AddFlushTaskEntry(std::unique_ptr &&entry) auto &last_task = pending_flush_work.back(); if (last_task->MergeFrom(std::move(flush_data_task))) { + LOG(INFO) << "AddFlushTaskEntry: Merge successful, task was " + "merged into last_task"; // Merge successful, task was merged into last_task flush_data_worker_ctx_.cv_.notify_all(); return; @@ -6057,7 +6071,7 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, << " quota: " << data_sync_mem_controller_.FlushMemoryQuota(); PostProcessFlushTaskEntries( - flush_task_entries, ckpt_err, &yield_fn, &resume_fn); + flush_task_entries, ckpt_err, &yield_fn, &resume_fn, &sync_yield_func); } void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk, @@ -6105,6 +6119,8 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) { if (!pending_flush_work.empty()) { + LOG(INFO) << "FlushDataWorker: flush work size = " + << pending_flush_work.size(); std::unique_ptr cur_work = std::move(pending_flush_work.front()); pending_flush_work.pop_front(); @@ -6354,7 +6370,8 @@ void LocalCcShards::RangeSplitWorker() bool LocalCcShards::UpdateStoreSlices( std::vector &flush_tasks, const std::function *yield_fptr, - const std::function *resume_fptr) + const std::function *resume_fptr, + const std::function *sync_yield_fptr) { std::vector update_range_slice_reqs; @@ -6398,7 +6415,7 @@ bool LocalCcShards::UpdateStoreSlices( if (!update_range_slice_reqs.empty()) { bool success = store_hd_->UpdateRangeSlices( - update_range_slice_reqs, yield_fptr, resume_fptr); + update_range_slice_reqs, yield_fptr, resume_fptr, sync_yield_fptr); return success; } From d8196bab4f1256ead764a9f01d6904eaea092754 Mon Sep 17 00:00:00 2001 From: lokax Date: Sat, 28 Feb 2026 18:09:34 +0800 Subject: [PATCH 18/38] map buffer --- tx_service/src/cc/local_cc_shards.cpp | 126 +++++++++++++------------- 1 file changed, 64 insertions(+), 62 deletions(-) diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 6f5365d1..16617953 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -285,23 +285,20 @@ LocalCcShards::~LocalCcShards() void LocalCcShards::StartBackgroudWorkers() { - // Ensure FlushDataWorker count matches DataSyncWorker count - // Note: worker_num_ is const, so we can't modify it. We use - // data_sync_worker_ctx_.worker_num_ to initialize the vectors and assert - // they match. - // Initialize per-worker data structures - const size_t worker_num = flush_data_worker_ctx_.worker_num_; - // Calculate buffer size + // cur_flush_buffers_ sized by data_sync_worker_num (one buffer per data + // sync worker). pending_flush_work_ and resume_queue_ sized by flush worker + // num. + const size_t data_sync_worker_num = data_sync_worker_ctx_.worker_num_; const uint64_t buffer_size = - data_sync_mem_controller_.FlushMemoryQuota() / (worker_num); + data_sync_mem_controller_.FlushMemoryQuota() / data_sync_worker_num; cur_flush_buffers_.clear(); - for (size_t i = 0; i < worker_num; ++i) + for (size_t i = 0; i < data_sync_worker_num; ++i) { cur_flush_buffers_.emplace_back( std::make_unique(buffer_size)); } - pending_flush_work_.resize(worker_num); - resume_queue_.resize(worker_num); + pending_flush_work_.resize(flush_data_worker_ctx_.worker_num_); + resume_queue_.resize(flush_data_worker_ctx_.worker_num_); // Starts flush worker threads firstly. for (int id = 0; id < flush_data_worker_ctx_.worker_num_; id++) @@ -4770,11 +4767,10 @@ void LocalCcShards::DataSyncForHashPartition( auto updated_memory_per_partition = partition_number_this_core ? updated_memory / partition_number_this_core : 0; + const size_t data_sync_worker_id = static_cast( + data_sync_task->id_ % data_sync_worker_ctx_.worker_num_); const size_t flush_buffer_size = - cur_flush_buffers_[DataSyncWorkerToFlushDataWorker(static_cast( - data_sync_task->id_ % - data_sync_worker_ctx_.worker_num_))] - ->GetFlushBufferSize(); + cur_flush_buffers_[data_sync_worker_id]->GetFlushBufferSize(); const size_t partition_number_per_scan = std::max(1UL, @@ -5779,18 +5775,19 @@ size_t LocalCcShards::DataSyncWorkerToFlushDataWorker( void LocalCcShards::AddFlushTaskEntry(std::unique_ptr &&entry) { assert(cur_flush_buffers_.size() == - static_cast(flush_data_worker_ctx_.worker_num_)); + static_cast(data_sync_worker_ctx_.worker_num_)); assert(pending_flush_work_.size() == static_cast(flush_data_worker_ctx_.worker_num_)); assert(entry->data_sync_task_ != nullptr); - // Compute target buffer/queue: map data_sync task id to fixed flush worker const auto &data_sync_task = entry->data_sync_task_; - size_t target = DataSyncWorkerToFlushDataWorker(static_cast( - data_sync_task->id_ % data_sync_worker_ctx_.worker_num_)); + const size_t data_sync_worker_id = static_cast( + data_sync_task->id_ % data_sync_worker_ctx_.worker_num_); + const size_t flush_target = + DataSyncWorkerToFlushDataWorker(data_sync_worker_id); std::unique_lock worker_lk(flush_data_worker_ctx_.mux_); - auto &cur_flush_buffer = *cur_flush_buffers_[target]; + auto &cur_flush_buffer = *cur_flush_buffers_[data_sync_worker_id]; cur_flush_buffer.AddFlushTaskEntry(std::move(entry)); @@ -5798,7 +5795,7 @@ void LocalCcShards::AddFlushTaskEntry(std::unique_ptr &&entry) cur_flush_buffer.MoveFlushData(false); if (flush_data_task != nullptr) { - auto &pending_flush_work = pending_flush_work_[target]; + auto &pending_flush_work = pending_flush_work_[flush_target]; // Try to merge with the last task if queue is not empty if (!pending_flush_work.empty()) @@ -5815,10 +5812,12 @@ void LocalCcShards::AddFlushTaskEntry(std::unique_ptr &&entry) } // Could not merge, wait if queue is full + /* while (pending_flush_work.size() >= 2) { flush_data_worker_ctx_.cv_.wait(worker_lk); } + */ // Add as new task pending_flush_work.emplace_back(std::move(flush_data_task)); @@ -5828,16 +5827,18 @@ void LocalCcShards::AddFlushTaskEntry(std::unique_ptr &&entry) void LocalCcShards::FlushCurrentFlushBuffer() { - assert(cur_flush_buffers_.size() == flush_data_worker_ctx_.worker_num_); - assert(pending_flush_work_.size() == flush_data_worker_ctx_.worker_num_); + assert(cur_flush_buffers_.size() == + static_cast(data_sync_worker_ctx_.worker_num_)); + assert(pending_flush_work_.size() == + static_cast(flush_data_worker_ctx_.worker_num_)); std::unique_lock worker_lk(flush_data_worker_ctx_.mux_); - // Flush all workers' buffers - for (int worker_idx = 0; worker_idx < flush_data_worker_ctx_.worker_num_; - ++worker_idx) + for (int i = 0; i < data_sync_worker_ctx_.worker_num_; ++i) { - auto &cur_flush_buffer = *cur_flush_buffers_[worker_idx]; - auto &pending_flush_work = pending_flush_work_[worker_idx]; + auto &cur_flush_buffer = *cur_flush_buffers_[i]; + size_t flush_target = + DataSyncWorkerToFlushDataWorker(static_cast(i)); + auto &pending_flush_work = pending_flush_work_[flush_target]; std::unique_ptr flush_data_task = cur_flush_buffer.MoveFlushData(true); @@ -5849,6 +5850,9 @@ void LocalCcShards::FlushCurrentFlushBuffer() auto &last_task = pending_flush_work.back(); if (last_task->MergeFrom(std::move(flush_data_task))) { + LOG(INFO) << "FlushCurrentFlushBuffer: Merge successful, " + "task was " + "merged into last_task"; // Merge successful, task was merged into last_task flush_data_worker_ctx_.cv_.notify_all(); continue; @@ -6102,16 +6106,15 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) assert(worker_idx < static_cast(flush_data_worker_ctx_.worker_num_)); assert(worker_idx < static_cast(pending_flush_work_.size())); - assert(worker_idx < static_cast(cur_flush_buffers_.size())); assert(worker_idx < static_cast(resume_queue_.size())); auto &pending_flush_work = pending_flush_work_[worker_idx]; - auto &cur_flush_buffer = *cur_flush_buffers_[worker_idx]; auto &resume_queue = resume_queue_[worker_idx]; using clock = std::chrono::steady_clock; using continuation = boost::context::continuation; - size_t previous_flush_size = 0; + std::vector previous_flush_sizes(data_sync_worker_ctx_.worker_num_, + 0); auto previous_size_update_time = clock::now(); std::unique_lock flush_worker_lk(flush_data_worker_ctx_.mux_); @@ -6206,10 +6209,10 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) flush_worker_lk, 10s, [this, + worker_idx, &pending_flush_work, &resume_queue, - &cur_flush_buffer, - &previous_flush_size, + &previous_flush_sizes, &previous_size_update_time] { if (!pending_flush_work.empty() || !resume_queue.empty() || @@ -6220,42 +6223,41 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) auto current_time = std::chrono::steady_clock::now(); if (current_time - previous_size_update_time > 10s) { - size_t current_flush_size = - cur_flush_buffer.GetPendingFlushSize(); - bool flush_size_changed = - current_flush_size != previous_flush_size; - previous_flush_size = current_flush_size; previous_size_update_time = current_time; - if (!flush_size_changed && current_flush_size > 0) + for (int i = 0; i < data_sync_worker_ctx_.worker_num_; ++i) { - // data sync might be stuck due to lock conflict with - // DDL. Flush current flush buffer to release catalog - // read lock held by ongoing data sync tx, which might - // block the DDL. - std::unique_ptr flush_data_task = - cur_flush_buffer.MoveFlushData(true); - if (flush_data_task != nullptr) + if (DataSyncWorkerToFlushDataWorker( + static_cast(i)) != worker_idx) { - // Try to merge with the last task if queue is not - // empty Note: flush_worker_lk is already held here - // (inside condition variable predicate) - if (!pending_flush_work.empty()) + continue; + } + size_t current_flush_size = + cur_flush_buffers_[i]->GetPendingFlushSize(); + bool flush_size_changed = + current_flush_size != previous_flush_sizes[i]; + previous_flush_sizes[i] = current_flush_size; + if (!flush_size_changed && current_flush_size > 0) + { + // data sync might be stuck due to lock conflict + // with DDL. Flush buffer to release catalog read + // lock held by ongoing data sync tx. + std::unique_ptr flush_data_task = + cur_flush_buffers_[i]->MoveFlushData(true); + if (flush_data_task != nullptr) { - auto &last_task = pending_flush_work.back(); - if (last_task->MergeFrom( - std::move(flush_data_task))) + if (!pending_flush_work.empty()) { - // Merge successful, task was merged into - // last_task - return true; + auto &last_task = pending_flush_work.back(); + if (last_task->MergeFrom( + std::move(flush_data_task))) + { + return true; + } } + pending_flush_work.emplace_back( + std::move(flush_data_task)); + return true; } - - // Add as new task. We just checked that - // pending_flush_work is empty, - pending_flush_work.emplace_back( - std::move(flush_data_task)); - return true; } } } From 96330bed7ff68040616f948c797fb5898016e99c Mon Sep 17 00:00:00 2001 From: lokax Date: Tue, 3 Mar 2026 10:46:55 +0800 Subject: [PATCH 19/38] remove debug log --- store_handler/data_store_service_client.cpp | 38 +++++++------- .../data_store_service_client_closure.cpp | 30 +++++------ tx_service/include/cc/cc_req_misc.h | 12 ++--- tx_service/include/cc/local_cc_shards.h | 11 ++-- tx_service/src/cc/local_cc_shards.cpp | 50 +++++++++++-------- 5 files changed, 76 insertions(+), 65 deletions(-) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index aaeb9b07..2b511fa4 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -487,9 +487,9 @@ bool DataStoreServiceClient::PutAllImpl( if (sync_yield_fptr != nullptr && batch_writes_since_yield >= MAX_BATCH_WRITES_WITHOUT_YIELD) { - LOG(INFO) << "PutAllImpl: Before sync yield, this = " - << sync_putall << ", batch_writes_since_yield = " - << batch_writes_since_yield; + // LOG(INFO) << "PutAllImpl: Before sync yield, this = " + // << sync_putall << ", batch_writes_since_yield = " + // << batch_writes_since_yield; (*sync_yield_fptr)(); batch_writes_since_yield = 0; } @@ -504,9 +504,9 @@ bool DataStoreServiceClient::PutAllImpl( // Wait for all partitions to complete if (yield_fptr != nullptr && resume_fptr != nullptr) { - LOG(INFO) << "PutAllImpl: Before Wait, this = " << sync_putall; + // LOG(INFO) << "PutAllImpl: Before Wait, this = " << sync_putall; sync_putall->Wait(yield_fptr, resume_fptr); - LOG(INFO) << "PutAllImpl: After Wait, this = " << sync_putall; + // LOG(INFO) << "PutAllImpl: After Wait, this = " << sync_putall; } else { @@ -1854,9 +1854,9 @@ bool DataStoreServiceClient::UpdateRangeSlices( ++iterations_since_yield; if (iterations_since_yield >= MAX_ITERATIONS_WITHOUT_YIELD) { - LOG(INFO) << "UpdateRangeSlices: prepare Before sync yield"; + // LOG(INFO) << "UpdateRangeSlices: prepare Before sync yield"; (*sync_yield_fptr)(); - LOG(INFO) << "UpdateRangeSlices: prepare After sync yield"; + // LOG(INFO) << "UpdateRangeSlices: prepare After sync yield"; iterations_since_yield = 0; } } @@ -1865,10 +1865,10 @@ bool DataStoreServiceClient::UpdateRangeSlices( auto end_time = std::chrono::steady_clock::now(); auto duration = std::chrono::duration_cast( end_time - start_time); - LOG(INFO) << "UpdateRangeSlices: Prepare slice batches and accumulate " - "metadata for all ranges took " - << duration.count() << "us" - << ", req size = " << update_range_slice_reqs.size(); + // LOG(INFO) << "UpdateRangeSlices: Prepare slice batches and accumulate " + // "metadata for all ranges took " + // << duration.count() << "us" + // << ", req size = " << update_range_slice_reqs.size(); // 2- Dispatch slice batches for all ranges concurrently (shared // SyncConcurrentRequest) @@ -1894,18 +1894,18 @@ bool DataStoreServiceClient::UpdateRangeSlices( ++iterations_since_yield; if (iterations_since_yield >= MAX_ITERATIONS_WITHOUT_YIELD) { - LOG(INFO) << "UpdateRangeSlices: Before sync yield"; + // LOG(INFO) << "UpdateRangeSlices: Before sync yield"; (*sync_yield_fptr)(); - LOG(INFO) << "UpdateRangeSlices: After sync yield"; + // LOG(INFO) << "UpdateRangeSlices: After sync yield"; iterations_since_yield = 0; } } } - LOG(INFO) << "UpdateRangeSlices: before WaitAll slice sync"; + // LOG(INFO) << "UpdateRangeSlices: before WaitAll slice sync"; // 3- Wait for slice requests to complete slice_sync_concurrent->WaitForAll(); - LOG(INFO) << "UpdateRangeSlices: after WaitAll slice sync"; + // LOG(INFO) << "UpdateRangeSlices: after WaitAll slice sync"; if (slice_sync_concurrent->result_.error_code() != remote::DataStoreError::NO_ERROR) @@ -1960,10 +1960,10 @@ bool DataStoreServiceClient::UpdateRangeSlices( DispatchRangeMetadataBatches( kv_range_table_name, meta_acc, meta_sync_concurrent); - LOG(INFO) << "UpdateRangeSlices: before WaitAll meta sync"; + // LOG(INFO) << "UpdateRangeSlices: before WaitAll meta sync"; // 5- Wait for metadata requests to complete meta_sync_concurrent->WaitForAll(); - LOG(INFO) << "UpdateRangeSlices: after WaitAll meta sync"; + // LOG(INFO) << "UpdateRangeSlices: after WaitAll meta sync"; // 6- Check for errors if (meta_sync_concurrent->result_.error_code() != @@ -1989,9 +1989,9 @@ bool DataStoreServiceClient::UpdateRangeSlices( FlushData(kv_range_table_names, callback_data, &SyncCallback); if (yield_fptr != nullptr && resume_fptr != nullptr) { - LOG(INFO) << "UpdateRangeSlices: before Wait meta sync"; + // LOG(INFO) << "UpdateRangeSlices: before Wait meta sync"; callback_data->Wait(yield_fptr, resume_fptr); - LOG(INFO) << "UpdateRangeSlices: after Wait meta sync"; + // LOG(INFO) << "UpdateRangeSlices: after Wait meta sync"; } else { diff --git a/store_handler/data_store_service_client_closure.cpp b/store_handler/data_store_service_client_closure.cpp index 66bc141c..13cd69de 100644 --- a/store_handler/data_store_service_client_closure.cpp +++ b/store_handler/data_store_service_client_closure.cpp @@ -535,9 +535,9 @@ void SyncPutAllData::Wait(const std::function *yield_fn, { waiting_.store(true, std::memory_order_release); lk.unlock(); - LOG(INFO) << "SyncPutAllData Wait: Before yield, this = " << this; + // LOG(INFO) << "SyncPutAllData Wait: Before yield, this = " << this; (*yield_fn)(); - LOG(INFO) << "SynPutAllData Wait: After yield, this = " << this; + // LOG(INFO) << "SynPutAllData Wait: After yield, this = " << this; lk.lock(); waiting_.store(false, std::memory_order_release); } @@ -552,15 +552,17 @@ void SyncConcurrentRequest::WaitForCapacityAndIncrement() { waiting_.store(true, std::memory_order_release); lk.unlock(); - LOG(INFO) << "SyncConcurrentRequest WaitForCapacityAndIncrement: " - "Before yield, this = " - << this; + // LOG(INFO) << "SyncConcurrentRequest WaitForCapacityAndIncrement: + // " + // "Before yield, this = " + // << this; (*yield_fn_)(); - LOG(INFO) << "SyncConcurrentRequest WaitForCapacityAndIncrement: " - "After yield, this = " - << this; + // LOG(INFO) << "SyncConcurrentRequest WaitForCapacityAndIncrement: + // " + // "After yield, this = " + // << this; lk.lock(); waiting_.store(false, std::memory_order_release); } @@ -582,15 +584,15 @@ void SyncConcurrentRequest::WaitForAll() { waiting_.store(true, std::memory_order_release); lk.unlock(); - LOG(INFO) - << "SyncConcurrentRequest WaitForAll: Before yield, this = " - << this; + // LOG(INFO) + // << "SyncConcurrentRequest WaitForAll: Before yield, this = " + // << this; (*yield_fn_)(); - LOG(INFO) - << "SyncConcurrentRequest WaitForAll: After yield, this = " - << this; + // LOG(INFO) + // << "SyncConcurrentRequest WaitForAll: After yield, this = " + // << this; lk.lock(); waiting_.store(false, std::memory_order_release); } diff --git a/tx_service/include/cc/cc_req_misc.h b/tx_service/include/cc/cc_req_misc.h index 0f2f01a5..9d8835d2 100644 --- a/tx_service/include/cc/cc_req_misc.h +++ b/tx_service/include/cc/cc_req_misc.h @@ -907,9 +907,9 @@ struct WaitableCc : public RunOnTxProcessorCc { waiting_.store(true, std::memory_order_release); lk.unlock(); - LOG(INFO) << "WaitableCc Wait: Before yield"; + // LOG(INFO) << "WaitableCc Wait: Before yield"; (*yield_fn)(); - LOG(INFO) << "WaitableCc Wait: After Yield"; + // LOG(INFO) << "WaitableCc Wait: After Yield"; lk.lock(); waiting_.store(false, std::memory_order_release); } @@ -969,7 +969,7 @@ struct WaitableCc : public RunOnTxProcessorCc waiting_.store(false, std::memory_order_release); auto *fn = resume_fn_; lk.unlock(); - LOG(INFO) << "WaitableCc Execute: before resume"; + // LOG(INFO) << "WaitableCc Execute: before resume"; (*fn)(); } else if (resume_fn_ == nullptr) @@ -1067,7 +1067,7 @@ struct UpdateCceCkptTsCc : public CcRequestBase waiting_.store(false, std::memory_order_release); auto *fn = resume_fn_; lk.unlock(); - LOG(INFO) << "UpdateCceCkptTsCc SetFinished: before resume"; + // LOG(INFO) << "UpdateCceCkptTsCc SetFinished: before resume"; (*fn)(); } else if (resume_fn_ == nullptr) @@ -1099,9 +1099,9 @@ struct UpdateCceCkptTsCc : public CcRequestBase { waiting_.store(true, std::memory_order_release); lk.unlock(); - LOG(INFO) << "UpdateCceCkptTsCc Wait: Before yield"; + // LOG(INFO) << "UpdateCceCkptTsCc Wait: Before yield"; (*yield_fn)(); - LOG(INFO) << "UpdateCceCkptTsCc Wait: After Yield"; + // LOG(INFO) << "UpdateCceCkptTsCc Wait: After Yield"; lk.lock(); waiting_.store(false, std::memory_order_release); } diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index 081a5056..e11fcb20 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -92,7 +92,7 @@ struct CoroCtx { ~CoroCtx() { - LOG(INFO) << "CoroCtx destructor, this = " << this; + // LOG(INFO) << "CoroCtx destructor, this = " << this; } boost::context::continuation coro_; @@ -2489,10 +2489,11 @@ class LocalCcShards const TxKey *end_key, bool flush_res); - bool UpdateStoreSlices(std::vector &task, - const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr, - const std::function *sync_yield_fptr = nullptr); + bool UpdateStoreSlices( + std::vector &task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr, + const std::function *sync_yield_fptr = nullptr); bool GetNextRangePartitionId(const TableName &tablename, const TableSchema *table_schema, diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 16617953..9cd9eca0 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -5803,8 +5803,8 @@ void LocalCcShards::AddFlushTaskEntry(std::unique_ptr &&entry) auto &last_task = pending_flush_work.back(); if (last_task->MergeFrom(std::move(flush_data_task))) { - LOG(INFO) << "AddFlushTaskEntry: Merge successful, task was " - "merged into last_task"; + // LOG(INFO) << "AddFlushTaskEntry: Merge successful, task was " + // "merged into last_task"; // Merge successful, task was merged into last_task flush_data_worker_ctx_.cv_.notify_all(); return; @@ -5850,9 +5850,10 @@ void LocalCcShards::FlushCurrentFlushBuffer() auto &last_task = pending_flush_work.back(); if (last_task->MergeFrom(std::move(flush_data_task))) { - LOG(INFO) << "FlushCurrentFlushBuffer: Merge successful, " - "task was " - "merged into last_task"; + // LOG(INFO) << "FlushCurrentFlushBuffer: Merge successful, + // " + // "task was " + // "merged into last_task"; // Merge successful, task was merged into last_task flush_data_worker_ctx_.cv_.notify_all(); continue; @@ -5878,6 +5879,7 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, const std::function &yield_fn, const std::function &resume_fn) { + auto start_time = std::chrono::steady_clock::now(); auto &flush_task_entries = cur_work->flush_task_entries_; bool succ = true; @@ -6005,9 +6007,9 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, auto duration = std::chrono::duration_cast( stop_time - start_time); - LOG(INFO) << "FlushDataImpl: UpdateCceCkptTsCc duration = " - << duration.count() << "us" - << ", rec size = " << entry->data_sync_vec_->size(); + // LOG(INFO) << "FlushDataImpl: UpdateCceCkptTsCc duration = " + // << duration.count() << "us" + // << ", rec size = " << entry->data_sync_vec_->size(); if (cce_entries_map.size() > 0) { @@ -6030,10 +6032,11 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, if (iterations_since_yield >= MAX_ITERATIONS_WITHOUT_YIELD) { - LOG(INFO) - << "FlushDataImpl: Before sync yield, task " - "addr = " - << cur_work << ", worker idx = " << worker_idx; + // LOG(INFO) + // << "FlushDataImpl: Before sync yield, task " + // "addr = " + // << cur_work << ", worker idx = " << + // worker_idx; sync_yield_func(); iterations_since_yield = 0; } @@ -6045,8 +6048,8 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, } } - LOG(INFO) << "FlushDataImpl: start reset cc, task addr = " << cur_work - << ", worker idx = " << worker_idx; + // LOG(INFO) << "FlushDataImpl: start reset cc, task addr = " << cur_work + // << ", worker idx = " << worker_idx; // Notify cc shards that dirty data has been flushed. This will re-enqueue // kickout data cc reqs if there are any. WaitableCc reset_cc( @@ -6076,6 +6079,10 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, PostProcessFlushTaskEntries( flush_task_entries, ckpt_err, &yield_fn, &resume_fn, &sync_yield_func); + auto stop_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast( + stop_time - start_time); + LOG(INFO) << "FlushDataImpl duration = " << duration.count() << "us"; } void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk, @@ -6122,8 +6129,8 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) { if (!pending_flush_work.empty()) { - LOG(INFO) << "FlushDataWorker: flush work size = " - << pending_flush_work.size(); + // LOG(INFO) << "FlushDataWorker: flush work size = " + // << pending_flush_work.size(); std::unique_ptr cur_work = std::move(pending_flush_work.front()); pending_flush_work.pop_front(); @@ -6139,7 +6146,7 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) { ctx->yield_fn = [&sink]() { - LOG(INFO) << "CoroCtx yield_fn"; + // LOG(INFO) << "CoroCtx yield_fn"; sink = sink.resume(); }; ctx->sync_yield_func = @@ -6153,8 +6160,8 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) { std::lock_guard lk( flush_data_worker_ctx_.mux_); - LOG(INFO) << "CoroCtx sync_yield_func: push " - "ctx to resume_queue"; + // LOG(INFO) << "CoroCtx sync_yield_func: push " + // "ctx to resume_queue"; resume_queue_[worker_idx].push_back( std::move(c)); flush_data_worker_ctx_.cv_.notify_all(); @@ -6168,8 +6175,9 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) { if (auto c = weak_ctx.lock()) { - LOG(INFO) - << "CoroCtx resume_fn: coro ctx = " << c.get(); + // LOG(INFO) + // << "CoroCtx resume_fn: coro ctx = " << + // c.get(); std::lock_guard lk( flush_data_worker_ctx_.mux_); resume_queue_[worker_idx].push_back(std::move(c)); From 9d471184e0b46d73d26e900000257f46353808d6 Mon Sep 17 00:00:00 2001 From: lokax Date: Tue, 3 Mar 2026 16:11:37 +0800 Subject: [PATCH 20/38] log --- store_handler/data_store_service_client.cpp | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index 2b511fa4..79591a03 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -330,6 +330,7 @@ bool DataStoreServiceClient::PutAllImpl( const std::function *resume_fptr, const std::function *sync_yield_fptr) { + auto prepare_start = std::chrono::steady_clock::now(); DLOG(INFO) << "DataStoreServiceClient::PutAll called with " << flush_task.size() << " tables to flush."; uint64_t now = txservice::LocalCcShards::ClockTsInMillseconds(); @@ -445,6 +446,8 @@ bool DataStoreServiceClient::PutAllImpl( range_partitions_map.clear(); } + auto prepare_end = std::chrono::steady_clock::now(); + // Set up global coordinator sync_putall->total_partitions_ = sync_putall->partition_states_.size(); @@ -459,6 +462,8 @@ bool DataStoreServiceClient::PutAllImpl( constexpr size_t MAX_BATCH_WRITES_WITHOUT_YIELD = 10; size_t batch_writes_since_yield = 0; + auto write_start = std::chrono::steady_clock::now(); + for (size_t i = 0; i < callback_data_list.size(); ++i) { auto *partition_state = sync_putall->partition_states_[i]; @@ -501,6 +506,9 @@ bool DataStoreServiceClient::PutAllImpl( } } + auto write_end = std::chrono::steady_clock::now(); + + auto wait_start = std::chrono::steady_clock::now(); // Wait for all partitions to complete if (yield_fptr != nullptr && resume_fptr != nullptr) { @@ -513,6 +521,23 @@ bool DataStoreServiceClient::PutAllImpl( sync_putall->Wait(); } + auto wait_end = std::chrono::steady_clock::now(); + LOG(INFO) << "PutAll(...) prepare duration = " + << std::chrono::duration_cast( + prepare_end - prepare_start) + .count() + << "us" + << ", write duration = " + << std::chrono::duration_cast( + write_end - write_start) + .count() + << "us" + << ", wait duration = " + << std::chrono::duration_cast( + wait_end - wait_start) + .count() + << "us"; + // Check for errors for (auto &partition_state : sync_putall->partition_states_) { From b9751ba838710d4a9d5b243a93376581dbd9367d Mon Sep 17 00:00:00 2001 From: lokax Date: Mon, 9 Mar 2026 15:43:00 +0800 Subject: [PATCH 21/38] rocksdb handle --- store_handler/rocksdb_handler.cpp | 281 ++++++++++++++++++++++++++++-- 1 file changed, 265 insertions(+), 16 deletions(-) diff --git a/store_handler/rocksdb_handler.cpp b/store_handler/rocksdb_handler.cpp index 981a2a62..19d0f9b2 100644 --- a/store_handler/rocksdb_handler.cpp +++ b/store_handler/rocksdb_handler.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -87,6 +88,83 @@ namespace EloqKV namespace { +/** + * Minimal sync callback for RocksDB PutAll/PersistKV yield/resume offload. + * Mirrors SyncCallbackData pattern from data_store_service_client_closure.h. + */ +struct RocksDBWriteSyncCallback +{ + bthread::Mutex mtx_; + bthread::ConditionVariable cv_; + bool finished_{false}; + bool success_{true}; + const std::function *yield_fn_{nullptr}; + const std::function *resume_fn_{nullptr}; + std::atomic waiting_{false}; + + void Reset() + { + finished_ = false; + success_ = true; + waiting_.store(false); + yield_fn_ = nullptr; + resume_fn_ = nullptr; + } + + void SetCoroCallbacks(const std::function *yield_fn, + const std::function *resume_fn) + { + yield_fn_ = yield_fn; + resume_fn_ = resume_fn; + } + + void Notify(bool ok) + { + std::unique_lock lk(mtx_); + success_ = ok; + finished_ = true; + if (resume_fn_ != nullptr && waiting_.load(std::memory_order_acquire)) + { + waiting_.store(false, std::memory_order_release); + auto *fn = resume_fn_; + lk.unlock(); + (*fn)(); + } + else if (resume_fn_ == nullptr) + { + cv_.notify_one(); + } + } + + void Wait() + { + std::unique_lock lk(mtx_); + while (!finished_) + { + cv_.wait(lk); + } + } + + void Wait(const std::function *yield_fn, + const std::function *resume_fn) + { + if (yield_fn == nullptr || resume_fn == nullptr) + { + Wait(); + return; + } + std::unique_lock lk(mtx_); + while (!finished_) + { + waiting_.store(true, std::memory_order_release); + lk.unlock(); + (*yield_fn)(); + lk.lock(); + waiting_.store(false, std::memory_order_release); + } + } +}; + std::string NormalizeDbPath(const std::string &path) { std::string normalized = path; @@ -438,14 +516,142 @@ bool RocksDBHandler::PutAll( const std::function *resume_fptr, const std::function *sync_yield_fptr) { - (void)yield_fptr; - (void)resume_fptr; - (void)sync_yield_fptr; + (void) sync_yield_fptr; std::thread::id this_id = std::this_thread::get_id(); if (batch.empty()) { return true; } + + if (yield_fptr != nullptr && resume_fptr != nullptr) + { + RocksDBWriteSyncCallback callback; + callback.Reset(); + callback.SetCoroCallbacks(yield_fptr, resume_fptr); + + query_worker_pool_->SubmitWork( + [this, &batch, &callback, this_id](size_t) + { + std::shared_lock db_lk(db_mux_); + auto db = GetDBPtr(); + if (!db) + { + callback.Notify(false); + return; + } + rocksdb::WriteOptions write_options; + write_options.disableWAL = true; + write_options.no_slowdown = false; + rocksdb::WriteBatch write_batch; + uint64_t write_batch_size = 0; + + for (auto &[kv_cf_name, flush_task_entries] : batch) + { + rocksdb::ColumnFamilyHandle *cfh = + GetColumnFamilyHandler(kv_cf_name.data()); + if (cfh == nullptr) + { + LOG(ERROR) << "Failed to get column family, cf name: " + << kv_cf_name; + callback.Notify(false); + return; + } + uint64_t now = + txservice::LocalCcShards::ClockTsInMillseconds(); + for (auto &flush_task_entry : flush_task_entries) + { + for (auto &flush_rec : + *flush_task_entry->data_sync_vec_) + { + txservice::TxKey key = flush_rec.Key(); + std::string rocksdb_key; + EncodeToKvKey(key, rocksdb_key); + + if (flush_rec.payload_status_ == + txservice::RecordStatus::Normal && + flush_rec.Payload()->GetTTL() > now) + { + std::vector rec_buf; + SerializeFlushRecord(flush_rec, rec_buf); + write_batch_size += rocksdb_key.size(); + write_batch_size += rec_buf.size(); + write_batch.Put( + cfh, + rocksdb::Slice(rocksdb_key.data(), + rocksdb_key.size()), + rocksdb::Slice(rec_buf.data(), + rec_buf.size())); + } + else + { + write_batch_size += rocksdb_key.size(); + write_batch.Delete( + cfh, + rocksdb::Slice(rocksdb_key.data(), + rocksdb_key.size())); + } + + if (write_batch_size >= batch_write_size_) + { + auto status = + db->Write(write_options, &write_batch); + if (!status.ok()) + { + LOG(ERROR) + << "PutAll end failed " + << ", thread id: " << this_id + << ", result:" + << static_cast(status.ok()) + << ", batch size:" << batch.size() + << ", error: " << status.ToString() + << ", error code: " << status.code(); + callback.Notify(false); + return; + } + if (metrics::enable_kv_metrics) + { + metrics::kv_meter->Collect( + metrics::NAME_KV_FLUSH_ROWS_TOTAL, + write_batch.Count(), + "base"); + } + write_batch.Clear(); + write_batch_size = 0; + } + } + } + } + + if (write_batch_size > 0) + { + auto status = db->Write(write_options, &write_batch); + if (!status.ok()) + { + LOG(ERROR) + << "PutAll end failed " + << ", thread id: " << this_id + << ", result:" << static_cast(status.ok()) + << ", batch size:" << batch.size() + << ", error: " << status.ToString() + << ", error code: " << status.code(); + callback.Notify(false); + return; + } + if (metrics::enable_kv_metrics) + { + metrics::kv_meter->Collect( + metrics::NAME_KV_FLUSH_ROWS_TOTAL, + write_batch.Count(), + "base"); + } + } + callback.Notify(true); + }); + + callback.Wait(yield_fptr, resume_fptr); + return callback.success_; + } + std::shared_lock db_lk(db_mux_); auto db = GetDBPtr(); if (!db) @@ -551,13 +757,56 @@ bool RocksDBHandler::PutAll( return true; } -bool RocksDBHandler::PersistKV( - const std::vector &kv_table_names, - const std::function *yield_fptr, - const std::function *resume_fptr) +bool RocksDBHandler::PersistKV(const std::vector &kv_table_names, + const std::function *yield_fptr, + const std::function *resume_fptr) { - (void)yield_fptr; - (void)resume_fptr; + if (yield_fptr != nullptr && resume_fptr != nullptr) + { + RocksDBWriteSyncCallback callback; + callback.Reset(); + callback.SetCoroCallbacks(yield_fptr, resume_fptr); + + query_worker_pool_->SubmitWork( + [this, &kv_table_names, &callback](size_t) + { + std::shared_lock db_lk(db_mux_); + auto db = GetDBPtr(); + if (!db) + { + callback.Notify(false); + return; + } + for (const std::string &kv_cf_name : kv_table_names) + { + rocksdb::ColumnFamilyHandle *cfh = + GetColumnFamilyHandler(kv_cf_name); + if (cfh == nullptr) + { + LOG(ERROR) << "Failed to get column family, cf name: " + << kv_cf_name; + callback.Notify(false); + return; + } + rocksdb::FlushOptions flush_options; + flush_options.allow_write_stall = true; + flush_options.wait = true; + auto status = db->Flush(flush_options, cfh); + if (!status.ok()) + { + LOG(ERROR) << "Unable to flush db with error: " + << status.ToString(); + callback.Notify(false); + return; + } + } + callback.Notify(true); + }); + + callback.Wait(yield_fptr, resume_fptr); + return callback.success_; + } + std::shared_lock db_lk(db_mux_); auto db = GetDBPtr(); if (!db) @@ -1528,9 +1777,9 @@ bool RocksDBHandler::UpdateRangeSlices( const std::function *resume_fptr, const std::function *sync_yield_fptr) { - (void)yield_fptr; - (void)resume_fptr; - (void)sync_yield_fptr; + (void) yield_fptr; + (void) resume_fptr; + (void) sync_yield_fptr; LOG(ERROR) << "RocksDBHandler::UpdateRangeSlices not implemented"; // Not implemented assert(false); @@ -1684,8 +1933,8 @@ bool RocksDBHandler::PutArchivesAll( const std::function *yield_fptr, const std::function *resume_fptr) { - (void)yield_fptr; - (void)resume_fptr; + (void) yield_fptr; + (void) resume_fptr; LOG(ERROR) << "RocksDBHandler::PutArchivesAll not implemented"; // Not implemented assert(false); @@ -1701,8 +1950,8 @@ bool RocksDBHandler::CopyBaseToArchive( const std::function *yield_fptr, const std::function *resume_fptr) { - (void)yield_fptr; - (void)resume_fptr; + (void) yield_fptr; + (void) resume_fptr; LOG(ERROR) << "RocksDBHandler::CopyBaseToArchive not implemented"; // Not implemented assert(false); From 90c191be5320a881f8621159e8e799bf4c9f026c Mon Sep 17 00:00:00 2001 From: lokax Date: Mon, 9 Mar 2026 17:21:47 +0800 Subject: [PATCH 22/38] fix heap order --- store_handler/bigtable_handler.cpp | 26 +++++++++++++++++++++---- tx_service/include/cc/local_cc_shards.h | 24 ++++++++++++++++++++--- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/store_handler/bigtable_handler.cpp b/store_handler/bigtable_handler.cpp index 172c321f..5b51b829 100644 --- a/store_handler/bigtable_handler.cpp +++ b/store_handler/bigtable_handler.cpp @@ -613,7 +613,9 @@ void EloqDS::BigTableHandler::FetchTableRanges( LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); std::unique_lock heap_lk( shards->table_ranges_heap_mux_); - mi_override_thread(shards->GetTableRangesHeapThreadId()); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = + mi_override_thread(shards->GetTableRangesHeapThreadId()); mi_heap_t *prev_heap = mi_heap_set_default(shards->GetTableRangesHeap()); @@ -622,7 +624,14 @@ void EloqDS::BigTableHandler::FetchTableRanges( mono_key.size()); mi_heap_set_default(prev_heap); - mi_restore_default_thread_id(); + if (is_override_thd) + { + mi_override_thread(prev_thd); + } + else + { + mi_restore_default_thread_id(); + } } int32_t partition_id = @@ -758,7 +767,9 @@ void EloqDS::BigTableHandler::OnFetchRangeSlices( LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); std::unique_lock heap_lk( shards->table_ranges_heap_mux_); - mi_override_thread(shards->GetTableRangesHeapThreadId()); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = + mi_override_thread(shards->GetTableRangesHeapThreadId()); mi_heap_t *prev_heap = mi_heap_set_default(shards->GetTableRangesHeap()); @@ -773,7 +784,14 @@ void EloqDS::BigTableHandler::OnFetchRangeSlices( } mi_heap_set_default(prev_heap); - mi_restore_default_thread_id(); + if (is_override_thd) + { + mi_override_thread(prev_thd); + } + else + { + mi_restore_default_thread_id(); + } heap_lk.unlock(); } diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index e11fcb20..24f69606 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -875,9 +875,27 @@ class LocalCcShards new_range_entries.push_back(new_range); } } - if (TableRangesMemoryFull()) { - KickoutRangeSlices(); + std::unique_lock heap_lk(table_ranges_heap_mux_); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = + mi_override_thread(GetTableRangesHeapThreadId()); + mi_heap_t *prev_heap = mi_heap_set_default(GetTableRangesHeap()); + bool range_slice_mem_full = TableRangesMemoryFull(); + mi_heap_set_default(prev_heap); + if (is_override_thd) + { + mi_override_thread(prev_thd); + } + else + { + mi_restore_default_thread_id(); + } + heap_lk.unlock(); + if (range_slice_mem_full) + { + KickoutRangeSlices(); + } } return new_range_entries; } @@ -1034,6 +1052,7 @@ class LocalCcShards range_entry->UpdateRangeEntry(version, std::move(range_slices)); } + mi_heap_set_default(prev_heap); if (is_override_thd) { mi_override_thread(prev_thd); @@ -1042,7 +1061,6 @@ class LocalCcShards { mi_restore_default_thread_id(); } - mi_heap_set_default(prev_heap); #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena_id); From 626ee0d23b16b4215e6acd492f4f4a9575b5f697 Mon Sep 17 00:00:00 2001 From: lokax Date: Wed, 11 Mar 2026 14:20:59 +0800 Subject: [PATCH 23/38] debug log --- tx_service/include/fault/log_replay_service.h | 5 +++- tx_service/src/cc/local_cc_shards.cpp | 5 ++++ tx_service/src/tx_index_operation.cpp | 24 +++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/tx_service/include/fault/log_replay_service.h b/tx_service/include/fault/log_replay_service.h index 6dcc9677..f89bfe7d 100644 --- a/tx_service/include/fault/log_replay_service.h +++ b/tx_service/include/fault/log_replay_service.h @@ -119,7 +119,10 @@ class RecoveryService : public brpc::StreamInputHandler, TxLog *log_agent, std::string ip, uint16_t port); - ~RecoveryService() = default; + ~RecoveryService() + { + DLOG(INFO) << "RecoveryService destructor"; + } void Shutdown(); diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 9cd9eca0..cab99098 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -2328,11 +2328,16 @@ std::pair LocalCcShards::PinStoreRange( ->BucketOwner() == ng_id; }()); + DLOG(INFO) << "fetch range slices, ng id = " << ng_id + << ", ng term = " << ng_term + << ", range table name = " << range_table_name.StringView(); // Fetch range slices info from data store if it's not loaded yet. range_entry->FetchRangeSlices( range_table_name, cc_request, ng_id, ng_term, cc_shard); return {range_entry, nullptr}; } + + DLOG(INFO) << "store range slice count = " << store_range->SlicesCount(); return {range_entry, store_range}; } diff --git a/tx_service/src/tx_index_operation.cpp b/tx_service/src/tx_index_operation.cpp index 7ef05670..cebb4a87 100644 --- a/tx_service/src/tx_index_operation.cpp +++ b/tx_service/src/tx_index_operation.cpp @@ -462,6 +462,30 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) { if (txm->CheckLeaderTerm()) { + const CcEntryAddr *cluster_config_addr = nullptr; + const auto &meta_rset = txm->rw_set_.MetaDataReadSet(); + for (const auto &[cce_addr, read_entry_pair] : meta_rset) + { + if (read_entry_pair.second == cluster_config_ccm_name_sv) + { + cluster_config_addr = &cce_addr; + break; + } + } + LOG(INFO) + << "Alter Table Index transaction unlock cluster config " + "lock failed, txn: " + << txm->TxNumber() << ", error code: " + << (int) unlock_cluster_config_op_.hd_result_.ErrorCode() + << ", error message: " + << unlock_cluster_config_op_.hd_result_.ErrorMsg() + << ", cluster config addr term" + << cluster_config_addr->Term() + << ", cluster config addr node group id: " + << cluster_config_addr->NodeGroupId() + << ", node group term: " + << Sharder::Instance().LeaderTerm( + cluster_config_addr->NodeGroupId()); // Releasing a local read lock should never fail assert(false); } From 8085b55525ce0c3a7e931ae97737e89972ce451e Mon Sep 17 00:00:00 2001 From: lokax Date: Wed, 11 Mar 2026 18:00:12 +0800 Subject: [PATCH 24/38] add ctr dctr --- tx_service/include/cc/range_slice.h | 8 +++++++- tx_service/include/range_record.h | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tx_service/include/cc/range_slice.h b/tx_service/include/cc/range_slice.h index 0961534c..cd9e9087 100644 --- a/tx_service/include/cc/range_slice.h +++ b/tx_service/include/cc/range_slice.h @@ -888,9 +888,15 @@ class TemplateStoreRange : public StoreRange init_key_cache, empty_range ? init_key_cache : false); slices_.emplace_back(std::move(slice)); + LOG(INFO) << "StoreRange init: addr=" << static_cast(this) + << " partition_id=" << partition_id_; } - ~TemplateStoreRange() = default; + ~TemplateStoreRange() + { + LOG(INFO) << "StoreRange destructor: addr=" << static_cast(this) + << " partition_id=" << partition_id_; + } void SetRangeEndKey(const KeyT *end_key) { diff --git a/tx_service/include/range_record.h b/tx_service/include/range_record.h index 701b904d..ebbd0bcd 100644 --- a/tx_service/include/range_record.h +++ b/tx_service/include/range_record.h @@ -32,6 +32,8 @@ #include #include +#include "glog/logging.h" + // #include "cc_req_misc.h" #include "data_sync_task.h" #include "range_bucket_key_record.h" @@ -634,6 +636,9 @@ struct TemplateTableRangeEntry : public TableRangeEntry if (range_slices_) { assert(range_slices_->Pins() == 0); + LOG(INFO) << "StoreRange DropStoreRange: addr=" + << static_cast(range_slices_.get()) + << " partition_id=" << range_slices_->PartitionId(); range_slices_ = nullptr; } } From 8db09a865969483ec06ed0d42a2079efeaab0ab1 Mon Sep 17 00:00:00 2001 From: lokax Date: Thu, 12 Mar 2026 11:32:34 +0800 Subject: [PATCH 25/38] debug log --- tx_service/include/cc/catalog_cc_map.h | 8 ++++++++ tx_service/include/range_record.h | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/tx_service/include/cc/catalog_cc_map.h b/tx_service/include/cc/catalog_cc_map.h index a66f5699..02c87b96 100644 --- a/tx_service/include/cc/catalog_cc_map.h +++ b/tx_service/include/cc/catalog_cc_map.h @@ -463,6 +463,14 @@ class CatalogCcMap UINT64_MAX, false); + LOG(INFO) + << "catalog init range slices: addr=" + << static_cast( + range.second->RangeSlices()) + << " partition_id=" + << range.second->RangeSlices() + ->PartitionId(); + mi_heap_set_default(prev_heap); if (is_override_thd) { diff --git a/tx_service/include/range_record.h b/tx_service/include/range_record.h index ebbd0bcd..8c0a5399 100644 --- a/tx_service/include/range_record.h +++ b/tx_service/include/range_record.h @@ -601,6 +601,11 @@ struct TemplateTableRangeEntry : public TableRangeEntry range_slices->InitSlices(std::move(slices)); } range_slices_ = std::move(range_slices); + LOG(INFO) << "TemplateTableRangeEntry InitRangeSlices: addr=" + << static_cast(range_slices_.get()) + << " partition_id=" << range_slices_->PartitionId() + << ", start key = " + << range_slices_->RangeStartKey()->ToString(); } /** From bda648d3ac377e7b359964c9fdf2ea5c6baf8f02 Mon Sep 17 00:00:00 2001 From: lokax Date: Thu, 12 Mar 2026 12:04:05 +0800 Subject: [PATCH 26/38] debug log --- tx_service/include/range_record.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tx_service/include/range_record.h b/tx_service/include/range_record.h index 8c0a5399..e9d4a777 100644 --- a/tx_service/include/range_record.h +++ b/tx_service/include/range_record.h @@ -544,6 +544,13 @@ struct TemplateTableRangeEntry : public TableRangeEntry { } + ~TemplateTableRangeEntry() + { + LOG(INFO) << "TemplateTableRangeEntry destructor: addr=" + << static_cast(this) + << " partition_id=" << range_info_.PartitionId(); + } + void UpdateRangeEntry(uint64_t version_ts, std::unique_ptr> slices) { @@ -596,9 +603,21 @@ struct TemplateTableRangeEntry : public TableRangeEntry estimate_rec_size, has_dml_since_ddl); + LOG(INFO) << "Start init slice, TemplateTableRangeEntry " + "InitRangeSlices: addr=" + << static_cast(range_slices.get()) + << " partition_id=" << range_slices->PartitionId() + << ", start key = " + << range_slices->RangeStartKey()->ToString(); + if (!empty_range) { range_slices->InitSlices(std::move(slices)); + LOG(INFO) << "TemplateTableRangeEntry InitRangeSlices: addr=" + << static_cast(range_slices.get()) + << " partition_id=" << range_slices->PartitionId() + << ", start key = " + << range_slices->RangeStartKey()->ToString(); } range_slices_ = std::move(range_slices); LOG(INFO) << "TemplateTableRangeEntry InitRangeSlices: addr=" From bf9952142f95f97a6e36bbf083759f8c758c62a3 Mon Sep 17 00:00:00 2001 From: lokax Date: Thu, 12 Mar 2026 17:28:09 +0800 Subject: [PATCH 27/38] remove range heap --- store_handler/bigtable_handler.cpp | 38 ++--- .../data_store_service_client_closure.cpp | 56 +++---- tx_service/include/cc/catalog_cc_map.h | 4 + tx_service/include/cc/local_cc_shards.h | 154 ++++++++++-------- tx_service/include/cc/range_cc_map.h | 28 ++-- tx_service/include/range_record.h | 13 +- tx_service/src/cc/cc_req_misc.cpp | 28 ++-- tx_service/src/cc/local_cc_shards.cpp | 60 ++++--- tx_service/src/cc/range_slice.cpp | 28 ++-- tx_service/src/fault/log_replay_service.cpp | 1 + 10 files changed, 216 insertions(+), 194 deletions(-) diff --git a/store_handler/bigtable_handler.cpp b/store_handler/bigtable_handler.cpp index 5b51b829..2bf89bc0 100644 --- a/store_handler/bigtable_handler.cpp +++ b/store_handler/bigtable_handler.cpp @@ -613,25 +613,25 @@ void EloqDS::BigTableHandler::FetchTableRanges( LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); std::unique_lock heap_lk( shards->table_ranges_heap_mux_); - bool is_override_thd = mi_is_override_thread(); - mi_threadid_t prev_thd = - mi_override_thread(shards->GetTableRangesHeapThreadId()); - mi_heap_t *prev_heap = - mi_heap_set_default(shards->GetTableRangesHeap()); + // bool is_override_thd = mi_is_override_thread(); + // mi_threadid_t prev_thd = + // mi_override_thread(shards->GetTableRangesHeapThreadId()); + // mi_heap_t *prev_heap = + // mi_heap_set_default(shards->GetTableRangesHeap()); start_key = std::make_unique( reinterpret_cast(mono_key.data()), mono_key.size()); - mi_heap_set_default(prev_heap); - if (is_override_thd) - { - mi_override_thread(prev_thd); - } - else - { - mi_restore_default_thread_id(); - } + // mi_heap_set_default(prev_heap); + // if (is_override_thd) + // { + // mi_override_thread(prev_thd); + // } + // else + // { + // mi_restore_default_thread_id(); + // } } int32_t partition_id = @@ -767,11 +767,11 @@ void EloqDS::BigTableHandler::OnFetchRangeSlices( LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); std::unique_lock heap_lk( shards->table_ranges_heap_mux_); - bool is_override_thd = mi_is_override_thread(); - mi_threadid_t prev_thd = - mi_override_thread(shards->GetTableRangesHeapThreadId()); - mi_heap_t *prev_heap = - mi_heap_set_default(shards->GetTableRangesHeap()); + // bool is_override_thd = mi_is_override_thread(); + // mi_threadid_t prev_thd = + // mi_override_thread(shards->GetTableRangesHeapThreadId()); + // mi_heap_t *prev_heap = + // mi_heap_set_default(shards->GetTableRangesHeap()); size_t offset = 0; while (offset < slice_keys_str.size()) diff --git a/store_handler/data_store_service_client_closure.cpp b/store_handler/data_store_service_client_closure.cpp index 13cd69de..85d46046 100644 --- a/store_handler/data_store_service_client_closure.cpp +++ b/store_handler/data_store_service_client_closure.cpp @@ -869,11 +869,11 @@ void FetchTableRangesCallback(void *data, txservice::LocalCcShards *shards = txservice::Sharder::Instance().GetLocalCcShards(); std::unique_lock heap_lk(shards->table_ranges_heap_mux_); - bool is_override_thd = mi_is_override_thread(); - mi_threadid_t prev_thd = - mi_override_thread(shards->GetTableRangesHeapThreadId()); - mi_heap_t *prev_heap = - mi_heap_set_default(shards->GetTableRangesHeap()); + // bool is_override_thd = mi_is_override_thread(); + // mi_threadid_t prev_thd = + // mi_override_thread(shards->GetTableRangesHeapThreadId()); + // mi_heap_t *prev_heap = + // mi_heap_set_default(shards->GetTableRangesHeap()); #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -933,15 +933,15 @@ void FetchTableRangesCallback(void *data, } } - mi_heap_set_default(prev_heap); - if (is_override_thd) - { - mi_override_thread(prev_thd); - } - else - { - mi_restore_default_thread_id(); - } + // mi_heap_set_default(prev_heap); + // if (is_override_thd) + // { + // mi_override_thread(prev_thd); + // } + // else + // { + // mi_restore_default_thread_id(); + // } #if defined(WITH_JEMALLOC) txservice::JemallocArenaSwitcher::SwitchToArena(prev_arena); @@ -1179,11 +1179,11 @@ void FetchRangeSlicesCallback(void *data, txservice::Sharder::Instance().GetLocalCcShards(); std::unique_lock heap_lk( shards->table_ranges_heap_mux_); - bool is_override_thd = mi_is_override_thread(); - mi_threadid_t prev_thd = - mi_override_thread(shards->GetTableRangesHeapThreadId()); - mi_heap_t *prev_heap = - mi_heap_set_default(shards->GetTableRangesHeap()); + // bool is_override_thd = mi_is_override_thread(); + // mi_threadid_t prev_thd = + // mi_override_thread(shards->GetTableRangesHeapThreadId()); + // mi_heap_t *prev_heap = + // mi_heap_set_default(shards->GetTableRangesHeap()); #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -1219,15 +1219,15 @@ void FetchRangeSlicesCallback(void *data, offset += (sizeof(uint32_t) + key_len + sizeof(uint32_t)); } - mi_heap_set_default(prev_heap); - if (is_override_thd) - { - mi_override_thread(prev_thd); - } - else - { - mi_restore_default_thread_id(); - } + // mi_heap_set_default(prev_heap); + // if (is_override_thd) + // { + // mi_override_thread(prev_thd); + // } + // else + // { + // mi_restore_default_thread_id(); + // } #if defined(WITH_JEMALLOC) txservice::JemallocArenaSwitcher::SwitchToArena(prev_arena); #endif diff --git a/tx_service/include/cc/catalog_cc_map.h b/tx_service/include/cc/catalog_cc_map.h index 02c87b96..d5fd1beb 100644 --- a/tx_service/include/cc/catalog_cc_map.h +++ b/tx_service/include/cc/catalog_cc_map.h @@ -434,6 +434,7 @@ class CatalogCcMap std::unique_lock heap_lk( shard_->local_shards_ .table_ranges_heap_mux_); + /* bool is_override_thd = mi_is_override_thread(); mi_threadid_t prev_thd = mi_override_thread( @@ -442,6 +443,7 @@ class CatalogCcMap mi_heap_t *prev_heap = mi_heap_set_default( shard_->local_shards_ .GetTableRangesHeap()); + */ #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -471,6 +473,7 @@ class CatalogCcMap << range.second->RangeSlices() ->PartitionId(); + /* mi_heap_set_default(prev_heap); if (is_override_thd) { @@ -480,6 +483,7 @@ class CatalogCcMap { mi_restore_default_thread_id(); } + */ #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena( diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index 24f69606..aa155808 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -455,11 +455,12 @@ class LocalCcShards void InitializeTableRangesHeap() { std::unique_lock lk(table_ranges_heap_mux_); - if (!table_ranges_heap_) - { - table_ranges_thread_id_ = mi_thread_id(); - table_ranges_heap_ = mi_heap_new(); - } + // table_ranges_heap_ usage commented out - use default heap + // if (!table_ranges_heap_) + // { + // table_ranges_thread_id_ = mi_thread_id(); + // table_ranges_heap_ = mi_heap_new(); + // } #if defined(WITH_JEMALLOC) // create table ranges arena @@ -497,7 +498,9 @@ class LocalCcShards mi_heap_t *GetTableRangesHeap() const { - return table_ranges_heap_; + // table_ranges_heap_ usage commented out + // return table_ranges_heap_; + return nullptr; } uint32_t GetTableRangesArenaId() const @@ -520,17 +523,19 @@ class LocalCcShards return (static_cast(allocated) >= range_slice_memory_limit_); #else - if (table_ranges_heap_ != nullptr) - { - int64_t allocated, committed; - mi_thread_stats(&allocated, &committed); - return (static_cast(allocated) >= - range_slice_memory_limit_); - } - else - { - return false; - } + // table_ranges_heap_ usage commented out - use default heap + // if (table_ranges_heap_ != nullptr) + // { + // int64_t allocated, committed; + // mi_thread_stats(&allocated, &committed); + // return (static_cast(allocated) >= + // range_slice_memory_limit_); + // } + // else + // { + // return false; + // } + return false; #endif } @@ -551,17 +556,19 @@ class LocalCcShards return (static_cast(allocated) <= target_memory_size); #else - if (table_ranges_heap_ != nullptr) - { - size_t target_memory_size = range_slice_memory_limit_ / 10 * 9; - int64_t allocated, committed; - mi_thread_stats(&allocated, &committed); - return (static_cast(allocated) <= target_memory_size); - } - else - { - return false; - } + // table_ranges_heap_ usage commented out - use default heap + // if (table_ranges_heap_ != nullptr) + // { + // size_t target_memory_size = range_slice_memory_limit_ / 10 * 9; + // int64_t allocated, committed; + // mi_thread_stats(&allocated, &committed); + // return (static_cast(allocated) <= target_memory_size); + // } + // else + // { + // return false; + // } + return true; #endif } @@ -576,29 +583,31 @@ class LocalCcShards << (bool) (static_cast(allocated) >= range_slice_memory_limit_); #else - std::unique_lock heap_lk(table_ranges_heap_mux_); - bool is_override_thd = mi_is_override_thread(); - mi_threadid_t prev_thd = - mi_override_thread(GetTableRangesHeapThreadId()); - mi_heap_t *prev_heap = mi_heap_set_default(GetTableRangesHeap()); - - int64_t allocated, committed; - mi_thread_stats(&allocated, &committed); - LOG(INFO) << "Table range memory report: allocated " << allocated - << ", committed " << committed << ", full: " - << (bool) (static_cast(allocated) >= - range_slice_memory_limit_); - - mi_heap_set_default(prev_heap); - if (is_override_thd) - { - mi_override_thread(prev_thd); - } - else - { - mi_restore_default_thread_id(); - } - heap_lk.unlock(); + // table_ranges_heap_ usage commented out - use default heap + // std::unique_lock heap_lk(table_ranges_heap_mux_); + // bool is_override_thd = mi_is_override_thread(); + // mi_threadid_t prev_thd = + // mi_override_thread(GetTableRangesHeapThreadId()); + // mi_heap_t *prev_heap = mi_heap_set_default(GetTableRangesHeap()); + // + // int64_t allocated, committed; + // mi_thread_stats(&allocated, &committed); + // LOG(INFO) << "Table range memory report: allocated " << allocated + // << ", committed " << committed << ", full: " + // << (bool) (static_cast(allocated) >= + // range_slice_memory_limit_); + // + // mi_heap_set_default(prev_heap); + // if (is_override_thd) + // { + // mi_override_thread(prev_thd); + // } + // else + // { + // mi_restore_default_thread_id(); + // } + // heap_lk.unlock(); + LOG(INFO) << "Table range memory report: table_ranges_heap_ disabled"; #endif } @@ -877,20 +886,21 @@ class LocalCcShards } { std::unique_lock heap_lk(table_ranges_heap_mux_); - bool is_override_thd = mi_is_override_thread(); - mi_threadid_t prev_thd = - mi_override_thread(GetTableRangesHeapThreadId()); - mi_heap_t *prev_heap = mi_heap_set_default(GetTableRangesHeap()); + // bool is_override_thd = mi_is_override_thread(); + // mi_threadid_t prev_thd = + // mi_override_thread(GetTableRangesHeapThreadId()); + // table_ranges_heap_ usage commented out + // mi_heap_t *prev_heap = mi_heap_set_default(GetTableRangesHeap()); bool range_slice_mem_full = TableRangesMemoryFull(); - mi_heap_set_default(prev_heap); - if (is_override_thd) - { - mi_override_thread(prev_thd); - } - else - { - mi_restore_default_thread_id(); - } + // mi_heap_set_default(prev_heap); + // if (is_override_thd) + // { + // mi_override_thread(prev_thd); + // } + // else + // { + // mi_restore_default_thread_id(); + // } heap_lk.unlock(); if (range_slice_mem_full) { @@ -939,9 +949,10 @@ class LocalCcShards GetRangeOwnerInternal(partition_id, ng_id)->BucketOwner(); std::unique_lock heap_lk(table_ranges_heap_mux_); - bool is_override_thd = mi_is_override_thread(); - mi_threadid_t prev_thd = mi_override_thread(table_ranges_thread_id_); - mi_heap_t *prev_heap = mi_heap_set_default(table_ranges_heap_); + // bool is_override_thd = mi_is_override_thread(); + // mi_threadid_t prev_thd = mi_override_thread(table_ranges_thread_id_); + // table_ranges_heap_ usage commented out + // mi_heap_t *prev_heap = mi_heap_set_default(table_ranges_heap_); #if defined(WITH_JEMALLOC) uint32_t prev_arena_id; @@ -1004,7 +1015,8 @@ class LocalCcShards has_dml_since_ddl); } - mi_heap_set_default(prev_heap); + // mi_heap_set_default(prev_heap); + /* if (is_override_thd) { mi_override_thread(prev_thd); @@ -1013,6 +1025,7 @@ class LocalCcShards { mi_restore_default_thread_id(); } + */ #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena_id); @@ -1052,7 +1065,9 @@ class LocalCcShards range_entry->UpdateRangeEntry(version, std::move(range_slices)); } - mi_heap_set_default(prev_heap); + // table_ranges_heap_ usage commented out + // mi_heap_set_default(prev_heap); + /* if (is_override_thd) { mi_override_thread(prev_thd); @@ -1061,6 +1076,7 @@ class LocalCcShards { mi_restore_default_thread_id(); } + */ #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena_id); diff --git a/tx_service/include/cc/range_cc_map.h b/tx_service/include/cc/range_cc_map.h index c6a7767e..16cefe24 100644 --- a/tx_service/include/cc/range_cc_map.h +++ b/tx_service/include/cc/range_cc_map.h @@ -1131,11 +1131,11 @@ class RangeCcMap : public TemplateCcMap std::unique_lock heap_lk( shard_->local_shards_.table_ranges_heap_mux_); - bool is_override_thd = mi_is_override_thread(); - mi_threadid_t prev_thd = mi_override_thread( - shard_->local_shards_.GetTableRangesHeapThreadId()); - mi_heap_t *prev_heap = mi_heap_set_default( - shard_->local_shards_.GetTableRangesHeap()); + // bool is_override_thd = mi_is_override_thread(); + // mi_threadid_t prev_thd = mi_override_thread( + // shard_->local_shards_.GetTableRangesHeapThreadId()); + // mi_heap_t *prev_heap = mi_heap_set_default( + // shard_->local_shards_.GetTableRangesHeap()); #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -1150,15 +1150,15 @@ class RangeCcMap : public TemplateCcMap this->cc_ng_id_, this->table_name_.IsBase()); - mi_heap_set_default(prev_heap); - if (is_override_thd) - { - mi_override_thread(prev_thd); - } - else - { - mi_restore_default_thread_id(); - } + // mi_heap_set_default(prev_heap); + // if (is_override_thd) + // { + // mi_override_thread(prev_thd); + // } + // else + // { + // mi_restore_default_thread_id(); + // } #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena); diff --git a/tx_service/include/range_record.h b/tx_service/include/range_record.h index e9d4a777..cb3d11a8 100644 --- a/tx_service/include/range_record.h +++ b/tx_service/include/range_record.h @@ -546,9 +546,9 @@ struct TemplateTableRangeEntry : public TableRangeEntry ~TemplateTableRangeEntry() { - LOG(INFO) << "TemplateTableRangeEntry destructor: addr=" - << static_cast(this) - << " partition_id=" << range_info_.PartitionId(); + // LOG(INFO) << "TemplateTableRangeEntry destructor: addr=" + // << static_cast(this) + // << " partition_id=" << range_info_.PartitionId(); } void UpdateRangeEntry(uint64_t version_ts, @@ -603,13 +603,6 @@ struct TemplateTableRangeEntry : public TableRangeEntry estimate_rec_size, has_dml_since_ddl); - LOG(INFO) << "Start init slice, TemplateTableRangeEntry " - "InitRangeSlices: addr=" - << static_cast(range_slices.get()) - << " partition_id=" << range_slices->PartitionId() - << ", start key = " - << range_slices->RangeStartKey()->ToString(); - if (!empty_range) { range_slices->InitSlices(std::move(slices)); diff --git a/tx_service/src/cc/cc_req_misc.cpp b/tx_service/src/cc/cc_req_misc.cpp index 2d6dbf31..db5e88df 100644 --- a/tx_service/src/cc/cc_req_misc.cpp +++ b/tx_service/src/cc/cc_req_misc.cpp @@ -408,11 +408,11 @@ void FetchRangeSlicesReq::SetFinish(CcErrorCode err) assert(range_entry_->RangeSlices() == nullptr); std::unique_lock heap_lk(shards->table_ranges_heap_mux_); - bool is_override_thd = mi_is_override_thread(); - mi_threadid_t prev_thd = - mi_override_thread(shards->GetTableRangesHeapThreadId()); - mi_heap_t *prev_heap = - mi_heap_set_default(shards->GetTableRangesHeap()); + // bool is_override_thd = mi_is_override_thread(); + // mi_threadid_t prev_thd = + // mi_override_thread(shards->GetTableRangesHeapThreadId()); + // mi_heap_t *prev_heap = + // mi_heap_set_default(shards->GetTableRangesHeap()); #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -429,15 +429,15 @@ void FetchRangeSlicesReq::SetFinish(CcErrorCode err) estimate_rec_size); bool range_slice_mem_full = shards->TableRangesMemoryFull(); - mi_heap_set_default(prev_heap); - if (is_override_thd) - { - mi_override_thread(prev_thd); - } - else - { - mi_restore_default_thread_id(); - } + // mi_heap_set_default(prev_heap); + // if (is_override_thd) + // { + // mi_override_thread(prev_thd); + // } + // else + // { + // mi_restore_default_thread_id(); + // } #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena); diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index cab99098..30183482 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -1064,9 +1064,9 @@ void LocalCcShards::InitTableRanges(const TableName &range_table_name, std::unique_lock lk(fast_meta_data_mux_); std::unique_lock heap_lk(table_ranges_heap_mux_); - bool is_override_thd = mi_is_override_thread(); - mi_threadid_t prev_thd = mi_override_thread(table_ranges_thread_id_); - mi_heap_t *prev_heap = mi_heap_set_default(table_ranges_heap_); + // bool is_override_thd = mi_is_override_thread(); + // mi_threadid_t prev_thd = mi_override_thread(table_ranges_thread_id_); + // mi_heap_t *prev_heap = mi_heap_set_default(table_ranges_heap_); #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -1159,15 +1159,15 @@ void LocalCcShards::InitTableRanges(const TableName &range_table_name, } } - mi_heap_set_default(prev_heap); - if (is_override_thd) - { - mi_override_thread(prev_thd); - } - else - { - mi_restore_default_thread_id(); - } + // mi_heap_set_default(prev_heap); + // if (is_override_thd) + // { + // mi_override_thread(prev_thd); + // } + // else + // { + // mi_restore_default_thread_id(); + // } #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena); @@ -1337,11 +1337,11 @@ void LocalCcShards::KickoutRangeSlices() { std::unique_lock heap_lk( table_ranges_heap_mux_); - bool is_override_thd = mi_is_override_thread(); - mi_threadid_t prev_thd = mi_override_thread( - GetTableRangesHeapThreadId()); - mi_heap_t *prev_heap = - mi_heap_set_default(GetTableRangesHeap()); + // bool is_override_thd = mi_is_override_thread(); + // mi_threadid_t prev_thd = mi_override_thread( + // GetTableRangesHeapThreadId()); + // mi_heap_t *prev_heap = + // mi_heap_set_default(GetTableRangesHeap()); #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -1355,15 +1355,15 @@ void LocalCcShards::KickoutRangeSlices() range_entry->DropStoreRange(); bool has_enough_mem = HasEnoughTableRangesMemory(); - mi_heap_set_default(prev_heap); - if (is_override_thd) - { - mi_override_thread(prev_thd); - } - else - { - mi_restore_default_thread_id(); - } + // mi_heap_set_default(prev_heap); + // if (is_override_thd) + // { + // mi_override_thread(prev_thd); + // } + // else + // { + // mi_restore_default_thread_id(); + // } #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena); @@ -1401,10 +1401,12 @@ void LocalCcShards::KickoutRangeSlices() if (entry->IsStoreRangeFree()) { std::unique_lock heap_lk(table_ranges_heap_mux_); + /* bool is_override_thd = mi_is_override_thread(); mi_threadid_t prev_thd = mi_override_thread(GetTableRangesHeapThreadId()); mi_heap_t *prev_heap = mi_heap_set_default(GetTableRangesHeap()); + */ #if defined(WITH_JEMALLOC) uint32_t prev_arena; JemallocArenaSwitcher::ReadCurrentArena(prev_arena); @@ -1416,6 +1418,7 @@ void LocalCcShards::KickoutRangeSlices() entry->DropStoreRange(); bool has_enough_mem = HasEnoughTableRangesMemory(); + /* mi_heap_set_default(prev_heap); if (is_override_thd) { @@ -1425,6 +1428,7 @@ void LocalCcShards::KickoutRangeSlices() { mi_restore_default_thread_id(); } + */ #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena); @@ -2144,11 +2148,13 @@ bool LocalCcShards::DropStoreRangesInBucket(NodeGroupId ng_id, { std::unique_lock heap_lk( table_ranges_heap_mux_); + /* bool is_override_thd = mi_is_override_thread(); mi_threadid_t prev_thd = mi_override_thread(GetTableRangesHeapThreadId()); mi_heap_t *prev_heap = mi_heap_set_default(GetTableRangesHeap()); + */ #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -2161,6 +2167,7 @@ bool LocalCcShards::DropStoreRangesInBucket(NodeGroupId ng_id, entry->DropStoreRange(); + /* mi_heap_set_default(prev_heap); if (is_override_thd) { @@ -2170,6 +2177,7 @@ bool LocalCcShards::DropStoreRangesInBucket(NodeGroupId ng_id, { mi_restore_default_thread_id(); } + */ #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena); diff --git a/tx_service/src/cc/range_slice.cpp b/tx_service/src/cc/range_slice.cpp index 68cfa4b0..03eda6b1 100644 --- a/tx_service/src/cc/range_slice.cpp +++ b/tx_service/src/cc/range_slice.cpp @@ -510,11 +510,11 @@ void StoreRange::UpdateSliceSpec(StoreSlice *slice, std::unique_lock heap_lk( local_cc_shards_.table_ranges_heap_mux_); - bool is_override_thd = mi_is_override_thread(); - mi_threadid_t prev_thd = - mi_override_thread(local_cc_shards_.GetTableRangesHeapThreadId()); - mi_heap_t *prev_heap = - mi_heap_set_default(local_cc_shards_.GetTableRangesHeap()); + // bool is_override_thd = mi_is_override_thread(); + // mi_threadid_t prev_thd = + // mi_override_thread(local_cc_shards_.GetTableRangesHeapThreadId()); + // mi_heap_t *prev_heap = + // mi_heap_set_default(local_cc_shards_.GetTableRangesHeap()); #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -527,15 +527,15 @@ void StoreRange::UpdateSliceSpec(StoreSlice *slice, UpdateSlice(slice, split_keys); bool range_slice_mem_full = local_cc_shards_.TableRangesMemoryFull(); - mi_heap_set_default(prev_heap); - if (is_override_thd) - { - mi_override_thread(prev_thd); - } - else - { - mi_restore_default_thread_id(); - } + // mi_heap_set_default(prev_heap); + // if (is_override_thd) + // { + // mi_override_thread(prev_thd); + // } + // else + // { + // mi_restore_default_thread_id(); + // } #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena); #endif diff --git a/tx_service/src/fault/log_replay_service.cpp b/tx_service/src/fault/log_replay_service.cpp index 77c35ec5..53d5d277 100644 --- a/tx_service/src/fault/log_replay_service.cpp +++ b/tx_service/src/fault/log_replay_service.cpp @@ -873,6 +873,7 @@ void RecoveryService::on_closed(brpc::StreamId id) std::lock_guard lk(inbound_mux_); info = &inbound_connections_.find(id)->second; } + DLOG(INFO) << "replay service stream: " << id << " wait and clear requests"; WaitAndClearRequests(id, info->mux_, info->on_fly_cnt_, From cf0365ddc8f4663359e447a4278801ea95876e0a Mon Sep 17 00:00:00 2001 From: lokax Date: Fri, 13 Mar 2026 15:15:19 +0800 Subject: [PATCH 28/38] remove comment --- .../data_store_service_client_closure.cpp | 56 +++++++++---------- tx_service/include/cc/catalog_cc_map.h | 4 -- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/store_handler/data_store_service_client_closure.cpp b/store_handler/data_store_service_client_closure.cpp index 85d46046..13cd69de 100644 --- a/store_handler/data_store_service_client_closure.cpp +++ b/store_handler/data_store_service_client_closure.cpp @@ -869,11 +869,11 @@ void FetchTableRangesCallback(void *data, txservice::LocalCcShards *shards = txservice::Sharder::Instance().GetLocalCcShards(); std::unique_lock heap_lk(shards->table_ranges_heap_mux_); - // bool is_override_thd = mi_is_override_thread(); - // mi_threadid_t prev_thd = - // mi_override_thread(shards->GetTableRangesHeapThreadId()); - // mi_heap_t *prev_heap = - // mi_heap_set_default(shards->GetTableRangesHeap()); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = + mi_override_thread(shards->GetTableRangesHeapThreadId()); + mi_heap_t *prev_heap = + mi_heap_set_default(shards->GetTableRangesHeap()); #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -933,15 +933,15 @@ void FetchTableRangesCallback(void *data, } } - // mi_heap_set_default(prev_heap); - // if (is_override_thd) - // { - // mi_override_thread(prev_thd); - // } - // else - // { - // mi_restore_default_thread_id(); - // } + mi_heap_set_default(prev_heap); + if (is_override_thd) + { + mi_override_thread(prev_thd); + } + else + { + mi_restore_default_thread_id(); + } #if defined(WITH_JEMALLOC) txservice::JemallocArenaSwitcher::SwitchToArena(prev_arena); @@ -1179,11 +1179,11 @@ void FetchRangeSlicesCallback(void *data, txservice::Sharder::Instance().GetLocalCcShards(); std::unique_lock heap_lk( shards->table_ranges_heap_mux_); - // bool is_override_thd = mi_is_override_thread(); - // mi_threadid_t prev_thd = - // mi_override_thread(shards->GetTableRangesHeapThreadId()); - // mi_heap_t *prev_heap = - // mi_heap_set_default(shards->GetTableRangesHeap()); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = + mi_override_thread(shards->GetTableRangesHeapThreadId()); + mi_heap_t *prev_heap = + mi_heap_set_default(shards->GetTableRangesHeap()); #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -1219,15 +1219,15 @@ void FetchRangeSlicesCallback(void *data, offset += (sizeof(uint32_t) + key_len + sizeof(uint32_t)); } - // mi_heap_set_default(prev_heap); - // if (is_override_thd) - // { - // mi_override_thread(prev_thd); - // } - // else - // { - // mi_restore_default_thread_id(); - // } + mi_heap_set_default(prev_heap); + if (is_override_thd) + { + mi_override_thread(prev_thd); + } + else + { + mi_restore_default_thread_id(); + } #if defined(WITH_JEMALLOC) txservice::JemallocArenaSwitcher::SwitchToArena(prev_arena); #endif diff --git a/tx_service/include/cc/catalog_cc_map.h b/tx_service/include/cc/catalog_cc_map.h index d5fd1beb..02c87b96 100644 --- a/tx_service/include/cc/catalog_cc_map.h +++ b/tx_service/include/cc/catalog_cc_map.h @@ -434,7 +434,6 @@ class CatalogCcMap std::unique_lock heap_lk( shard_->local_shards_ .table_ranges_heap_mux_); - /* bool is_override_thd = mi_is_override_thread(); mi_threadid_t prev_thd = mi_override_thread( @@ -443,7 +442,6 @@ class CatalogCcMap mi_heap_t *prev_heap = mi_heap_set_default( shard_->local_shards_ .GetTableRangesHeap()); - */ #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -473,7 +471,6 @@ class CatalogCcMap << range.second->RangeSlices() ->PartitionId(); - /* mi_heap_set_default(prev_heap); if (is_override_thd) { @@ -483,7 +480,6 @@ class CatalogCcMap { mi_restore_default_thread_id(); } - */ #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena( From 1cdd8b0fbc03a68172110cf281bdbe3e816c1dcd Mon Sep 17 00:00:00 2001 From: lokax Date: Fri, 13 Mar 2026 16:39:19 +0800 Subject: [PATCH 29/38] debug log --- tx_service/include/cc/local_cc_shards.h | 14 +++++++------- tx_service/src/fault/log_replay_service.cpp | 6 ++++++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index aa155808..f44878c0 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -456,11 +456,11 @@ class LocalCcShards { std::unique_lock lk(table_ranges_heap_mux_); // table_ranges_heap_ usage commented out - use default heap - // if (!table_ranges_heap_) - // { - // table_ranges_thread_id_ = mi_thread_id(); - // table_ranges_heap_ = mi_heap_new(); - // } + if (!table_ranges_heap_) + { + table_ranges_thread_id_ = mi_thread_id(); + table_ranges_heap_ = mi_heap_new(); + } #if defined(WITH_JEMALLOC) // create table ranges arena @@ -499,8 +499,8 @@ class LocalCcShards mi_heap_t *GetTableRangesHeap() const { // table_ranges_heap_ usage commented out - // return table_ranges_heap_; - return nullptr; + return table_ranges_heap_; + // return nullptr; } uint32_t GetTableRangesArenaId() const diff --git a/tx_service/src/fault/log_replay_service.cpp b/tx_service/src/fault/log_replay_service.cpp index 53d5d277..5b71098a 100644 --- a/tx_service/src/fault/log_replay_service.cpp +++ b/tx_service/src/fault/log_replay_service.cpp @@ -287,6 +287,7 @@ void RecoveryService::Connect(::google::protobuf::RpcController *controller, // Invalid connection request. return; } + // node group is trying to recover cc_ng_term. Check if this log group // has finished replay. if (Sharder::Instance().CheckLogGroupReplayFinished( @@ -899,6 +900,8 @@ void RecoveryService::WaitAndClearRequests(brpc::StreamId stream_id, WaitingStatus waiting_status) { size_t on_fly_cnt = on_fly_cnt_.load(std::memory_order_relaxed); + DLOG(INFO) << "wait replay requests, stream id = " << stream_id + << ", on fly cnt = " << on_fly_cnt; if ((on_fly_cnt > 0 && waiting_status == WaitingStatus::WaitForAll) || (on_fly_cnt > 10000 && waiting_status == WaitingStatus::WaitForMany)) { @@ -914,6 +917,7 @@ void RecoveryService::WaitAndClearRequests(brpc::StreamId stream_id, } status.store(WaitingStatus::Active, std::memory_order_relaxed); } + DLOG(INFO) << "wait finished, stream id = " << stream_id; std::unique_lock lk(mux); if (recovery_error) { @@ -942,6 +946,8 @@ void RecoveryService::WaitAndClearRequests(brpc::StreamId stream_id, if (info.cc_ng_id_ == error_node_group_id && info.cc_ng_term_ == error_term) { + DLOG(INFO) << "RecoveryService, close stream, stream id = " + << stream_id; brpc::StreamClose(stream_id); } } From 2fe44a1a4f363f73796af3eef5c38e13aadf66d4 Mon Sep 17 00:00:00 2001 From: lokax Date: Fri, 13 Mar 2026 17:55:23 +0800 Subject: [PATCH 30/38] debug log --- tx_service/include/cc/range_slice.h | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tx_service/include/cc/range_slice.h b/tx_service/include/cc/range_slice.h index cd9e9087..6311f899 100644 --- a/tx_service/include/cc/range_slice.h +++ b/tx_service/include/cc/range_slice.h @@ -888,14 +888,23 @@ class TemplateStoreRange : public StoreRange init_key_cache, empty_range ? init_key_cache : false); slices_.emplace_back(std::move(slice)); + if (partition_id_ == 0 && partition_id != 0) + { + LOG(INFO) << "StoreRange init: addr=" << static_cast(this) + << " partition_id=" << partition_id_ + << ", part id = " << partition_id; + assert(false); + } LOG(INFO) << "StoreRange init: addr=" << static_cast(this) - << " partition_id=" << partition_id_; + << " partition_id=" << partition_id_ + << ", part id = " << partition_id; } ~TemplateStoreRange() { - LOG(INFO) << "StoreRange destructor: addr=" << static_cast(this) - << " partition_id=" << partition_id_; + // LOG(INFO) << "StoreRange destructor: addr=" << static_cast(this) + // << " partition_id=" << partition_id_; } void SetRangeEndKey(const KeyT *end_key) From 0c09d6c1edd29fe150a116a31ec4a307688550cd Mon Sep 17 00:00:00 2001 From: lokax Date: Mon, 16 Mar 2026 15:34:18 +0800 Subject: [PATCH 31/38] fix log replay service --- tx_service/include/cc/catalog_cc_map.h | 14 +++--- tx_service/include/cc/range_cc_map.h | 2 +- tx_service/include/cc/range_slice.h | 4 +- tx_service/include/fault/log_replay_service.h | 6 ++- tx_service/src/fault/log_replay_service.cpp | 46 +++++++++++-------- 5 files changed, 41 insertions(+), 31 deletions(-) diff --git a/tx_service/include/cc/catalog_cc_map.h b/tx_service/include/cc/catalog_cc_map.h index 02c87b96..3f995d06 100644 --- a/tx_service/include/cc/catalog_cc_map.h +++ b/tx_service/include/cc/catalog_cc_map.h @@ -1297,8 +1297,8 @@ class CatalogCcMap { LOG(INFO) << "ReplayLogCc, node_group(#" << req.NodeGroupId() << ") term < 0, tx:" << req.Txn(); - req.Result()->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); - return false; + req.AbortCcRequest(CcErrorCode::REQUESTED_NODE_NOT_LEADER); + return true; } ::txlog::SchemaOpMessage schema_op_msg; @@ -1359,7 +1359,7 @@ class CatalogCcMap << ", catalog entry of the same or higher " "version exists, stop replaying this schema op"; req.SetFinish(); - return false; + return true; } catalog_entry = new_catalog_entry; } @@ -1432,8 +1432,8 @@ class CatalogCcMap else if (!need_to_upsert_kv_table && upsert_kv_err_code.second != CcErrorCode::NO_ERROR) { - req.Result()->SetError(upsert_kv_err_code.second); - return false; + req.AbortCcRequest(upsert_kv_err_code.second); + return true; } auto [success, new_catalog_entry] = shard_->CreateCatalog( @@ -1455,7 +1455,7 @@ class CatalogCcMap << ", catalog entry of the same or higher version " "exists, stop replaying this schema op"; req.SetFinish(); - return false; + return true; } catalog_entry = new_catalog_entry; } @@ -1743,7 +1743,7 @@ class CatalogCcMap } } - return false; + return true; } bool Execute(BroadcastStatisticsCc &req) override diff --git a/tx_service/include/cc/range_cc_map.h b/tx_service/include/cc/range_cc_map.h index 16cefe24..28f00951 100644 --- a/tx_service/include/cc/range_cc_map.h +++ b/tx_service/include/cc/range_cc_map.h @@ -826,7 +826,7 @@ class RangeCcMap : public TemplateCcMap int64_t ng_term = Sharder::Instance().CandidateLeaderTerm(group_id); if (ng_term < 0) { - req.Result()->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); + req.AbortCcRequest(CcErrorCode::REQUESTED_NODE_NOT_LEADER); return true; } diff --git a/tx_service/include/cc/range_slice.h b/tx_service/include/cc/range_slice.h index 6311f899..75711b4e 100644 --- a/tx_service/include/cc/range_slice.h +++ b/tx_service/include/cc/range_slice.h @@ -902,8 +902,8 @@ class TemplateStoreRange : public StoreRange ~TemplateStoreRange() { - // LOG(INFO) << "StoreRange destructor: addr=" << static_cast(this) + LOG(INFO) << "StoreRange destructor: addr=" + << static_cast(this); // << " partition_id=" << partition_id_; } diff --git a/tx_service/include/fault/log_replay_service.h b/tx_service/include/fault/log_replay_service.h index f89bfe7d..fa63692f 100644 --- a/tx_service/include/fault/log_replay_service.h +++ b/tx_service/include/fault/log_replay_service.h @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -273,8 +274,9 @@ class RecoveryService : public brpc::StreamInputHandler, LocalCcShards &local_shards_; // Each ConnectionInfo is uniquely identified by - // pair. - std::unordered_map inbound_connections_; + // pair. unique_ptr ensures pointer stability across map rehashing. + std::unordered_map> + inbound_connections_; int active_stream_cnt_ = 0; bthread::Mutex inbound_mux_; bthread::ConditionVariable inbound_cv_; diff --git a/tx_service/src/fault/log_replay_service.cpp b/tx_service/src/fault/log_replay_service.cpp index 5b71098a..2d372eff 100644 --- a/tx_service/src/fault/log_replay_service.cpp +++ b/tx_service/src/fault/log_replay_service.cpp @@ -303,14 +303,14 @@ void RecoveryService::Connect(::google::protobuf::RpcController *controller, // Clean up old log group stream connection. for (auto &[stream_id, info] : inbound_connections_) { - if (info.cc_ng_id_ == cc_ng_id && - info.log_group_id_ == log_group_id) + if (info->cc_ng_id_ == cc_ng_id && + info->log_group_id_ == log_group_id) { - if (cc_ng_term > info.cc_ng_term_) + if (cc_ng_term > info->cc_ng_term_) { // cc_ng_term > info.cc_ng_term_, the cc_ng has failed over } - else if (cc_ng_term == info.cc_ng_term_) + else if (cc_ng_term == info->cc_ng_term_) { // the cc_ng is still recovering, and this request is a // response for RecoveryService's stream timeout and resend @@ -345,12 +345,10 @@ void RecoveryService::Connect(::google::protobuf::RpcController *controller, return; } - inbound_connections_.try_emplace(stream_socket, - log_group_id, - cc_ng_id, - cc_ng_term, - replay_start_ts, - recovering); + inbound_connections_.try_emplace( + stream_socket, + std::make_unique( + log_group_id, cc_ng_id, cc_ng_term, replay_start_ts, recovering)); // Initialize/refresh the global DDL barrier for recovering streams. if (recovering) @@ -442,7 +440,12 @@ int RecoveryService::on_received_messages(brpc::StreamId stream_id, NodeGroupReplayBarrier *barrier; { std::lock_guard lk(inbound_mux_); - info = &inbound_connections_.find(stream_id)->second; + auto it = inbound_connections_.find(stream_id); + if (it == inbound_connections_.end()) + { + return -1; + } + info = it->second.get(); barrier = GetReplayBarrier(info->cc_ng_id_); } bthread::Mutex &mux = info->mux_; @@ -837,7 +840,7 @@ void RecoveryService::on_idle_timeout(brpc::StreamId id) // this stream has been replaced by a newer one return; } - info = &it->second; + info = it->second.get(); } BAIDU_SCOPED_LOCK(info->mux_); if (info->recovering_) @@ -872,7 +875,12 @@ void RecoveryService::on_closed(brpc::StreamId id) ConnectionInfo *info; { std::lock_guard lk(inbound_mux_); - info = &inbound_connections_.find(id)->second; + auto it = inbound_connections_.find(id); + if (it == inbound_connections_.end()) + { + return; + } + info = it->second.get(); } DLOG(INFO) << "replay service stream: " << id << " wait and clear requests"; WaitAndClearRequests(id, @@ -933,18 +941,18 @@ void RecoveryService::WaitAndClearRequests(brpc::StreamId stream_id, return; } - error_node_group_id = it->second.cc_ng_id_; - error_term = it->second.cc_ng_term_; + error_node_group_id = it->second->cc_ng_id_; + error_term = it->second->cc_ng_term_; uint64_t replay_start_ts = 0; // close all the streams belonging to the current node group and term. for (const auto &[stream_id, info] : inbound_connections_) { assert(replay_start_ts == 0 || - replay_start_ts == info.replay_start_ts_); - replay_start_ts = info.replay_start_ts_; - if (info.cc_ng_id_ == error_node_group_id && - info.cc_ng_term_ == error_term) + replay_start_ts == info->replay_start_ts_); + replay_start_ts = info->replay_start_ts_; + if (info->cc_ng_id_ == error_node_group_id && + info->cc_ng_term_ == error_term) { DLOG(INFO) << "RecoveryService, close stream, stream id = " << stream_id; From ed3fd9a2af5474ca5c8bdcb801d3d2b1ca4233e3 Mon Sep 17 00:00:00 2001 From: lokax Date: Wed, 18 Mar 2026 11:06:50 +0800 Subject: [PATCH 32/38] fix build --- tx_service/include/cc/local_cc_shards.h | 3 --- tx_service/src/fault/log_replay_service.cpp | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index f44878c0..b16d415f 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -455,7 +455,6 @@ class LocalCcShards void InitializeTableRangesHeap() { std::unique_lock lk(table_ranges_heap_mux_); - // table_ranges_heap_ usage commented out - use default heap if (!table_ranges_heap_) { table_ranges_thread_id_ = mi_thread_id(); @@ -498,9 +497,7 @@ class LocalCcShards mi_heap_t *GetTableRangesHeap() const { - // table_ranges_heap_ usage commented out return table_ranges_heap_; - // return nullptr; } uint32_t GetTableRangesArenaId() const diff --git a/tx_service/src/fault/log_replay_service.cpp b/tx_service/src/fault/log_replay_service.cpp index 2d372eff..40b91123 100644 --- a/tx_service/src/fault/log_replay_service.cpp +++ b/tx_service/src/fault/log_replay_service.cpp @@ -128,10 +128,10 @@ RecoveryService::RecoveryService(LocalCcShards &local_shards, inbound_mux_); for (const auto &entry : inbound_connections_) { - if (entry.second.cc_ng_id_ == ng_id && - entry.second.cc_ng_term_ == + if (entry.second->cc_ng_id_ == ng_id && + entry.second->cc_ng_term_ == candidate_term && - entry.second.recovering_) + entry.second->recovering_) { has_replay_connection = true; break; From 85da27c4d779c3f8d7bf8f04761509db7c26395d Mon Sep 17 00:00:00 2001 From: lokax Date: Wed, 18 Mar 2026 17:36:44 +0800 Subject: [PATCH 33/38] remove log and heap --- store_handler/bigtable_handler.cpp | 38 ++--- store_handler/data_store_service_client.cpp | 39 ----- .../data_store_service_client_closure.h | 6 +- tx_service/include/cc/local_cc_shards.h | 151 +++++++++--------- tx_service/include/cc/range_cc_map.h | 28 ++-- tx_service/src/cc/cc_req_misc.cpp | 28 ++-- tx_service/src/cc/local_cc_shards.cpp | 94 ++++++----- tx_service/src/cc/range_slice.cpp | 28 ++-- 8 files changed, 197 insertions(+), 215 deletions(-) diff --git a/store_handler/bigtable_handler.cpp b/store_handler/bigtable_handler.cpp index 2bf89bc0..5b51b829 100644 --- a/store_handler/bigtable_handler.cpp +++ b/store_handler/bigtable_handler.cpp @@ -613,25 +613,25 @@ void EloqDS::BigTableHandler::FetchTableRanges( LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); std::unique_lock heap_lk( shards->table_ranges_heap_mux_); - // bool is_override_thd = mi_is_override_thread(); - // mi_threadid_t prev_thd = - // mi_override_thread(shards->GetTableRangesHeapThreadId()); - // mi_heap_t *prev_heap = - // mi_heap_set_default(shards->GetTableRangesHeap()); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = + mi_override_thread(shards->GetTableRangesHeapThreadId()); + mi_heap_t *prev_heap = + mi_heap_set_default(shards->GetTableRangesHeap()); start_key = std::make_unique( reinterpret_cast(mono_key.data()), mono_key.size()); - // mi_heap_set_default(prev_heap); - // if (is_override_thd) - // { - // mi_override_thread(prev_thd); - // } - // else - // { - // mi_restore_default_thread_id(); - // } + mi_heap_set_default(prev_heap); + if (is_override_thd) + { + mi_override_thread(prev_thd); + } + else + { + mi_restore_default_thread_id(); + } } int32_t partition_id = @@ -767,11 +767,11 @@ void EloqDS::BigTableHandler::OnFetchRangeSlices( LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); std::unique_lock heap_lk( shards->table_ranges_heap_mux_); - // bool is_override_thd = mi_is_override_thread(); - // mi_threadid_t prev_thd = - // mi_override_thread(shards->GetTableRangesHeapThreadId()); - // mi_heap_t *prev_heap = - // mi_heap_set_default(shards->GetTableRangesHeap()); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = + mi_override_thread(shards->GetTableRangesHeapThreadId()); + mi_heap_t *prev_heap = + mi_heap_set_default(shards->GetTableRangesHeap()); size_t offset = 0; while (offset < slice_keys_str.size()) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index 79591a03..3bd9f0fe 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -330,7 +330,6 @@ bool DataStoreServiceClient::PutAllImpl( const std::function *resume_fptr, const std::function *sync_yield_fptr) { - auto prepare_start = std::chrono::steady_clock::now(); DLOG(INFO) << "DataStoreServiceClient::PutAll called with " << flush_task.size() << " tables to flush."; uint64_t now = txservice::LocalCcShards::ClockTsInMillseconds(); @@ -446,8 +445,6 @@ bool DataStoreServiceClient::PutAllImpl( range_partitions_map.clear(); } - auto prepare_end = std::chrono::steady_clock::now(); - // Set up global coordinator sync_putall->total_partitions_ = sync_putall->partition_states_.size(); @@ -462,8 +459,6 @@ bool DataStoreServiceClient::PutAllImpl( constexpr size_t MAX_BATCH_WRITES_WITHOUT_YIELD = 10; size_t batch_writes_since_yield = 0; - auto write_start = std::chrono::steady_clock::now(); - for (size_t i = 0; i < callback_data_list.size(); ++i) { auto *partition_state = sync_putall->partition_states_[i]; @@ -492,9 +487,6 @@ bool DataStoreServiceClient::PutAllImpl( if (sync_yield_fptr != nullptr && batch_writes_since_yield >= MAX_BATCH_WRITES_WITHOUT_YIELD) { - // LOG(INFO) << "PutAllImpl: Before sync yield, this = " - // << sync_putall << ", batch_writes_since_yield = " - // << batch_writes_since_yield; (*sync_yield_fptr)(); batch_writes_since_yield = 0; } @@ -505,10 +497,6 @@ bool DataStoreServiceClient::PutAllImpl( sync_putall->OnPartitionCompleted(); } } - - auto write_end = std::chrono::steady_clock::now(); - - auto wait_start = std::chrono::steady_clock::now(); // Wait for all partitions to complete if (yield_fptr != nullptr && resume_fptr != nullptr) { @@ -521,23 +509,6 @@ bool DataStoreServiceClient::PutAllImpl( sync_putall->Wait(); } - auto wait_end = std::chrono::steady_clock::now(); - LOG(INFO) << "PutAll(...) prepare duration = " - << std::chrono::duration_cast( - prepare_end - prepare_start) - .count() - << "us" - << ", write duration = " - << std::chrono::duration_cast( - write_end - write_start) - .count() - << "us" - << ", wait duration = " - << std::chrono::duration_cast( - wait_end - wait_start) - .count() - << "us"; - // Check for errors for (auto &partition_state : sync_putall->partition_states_) { @@ -1831,8 +1802,6 @@ bool DataStoreServiceClient::UpdateRangeSlices( slice_plans.reserve(update_range_slice_reqs.size()); RangeMetadataAccumulator meta_acc; - auto start_time = std::chrono::steady_clock::now(); - constexpr size_t MAX_ITERATIONS_WITHOUT_YIELD = 10; size_t iterations_since_yield = 0; @@ -1887,14 +1856,6 @@ bool DataStoreServiceClient::UpdateRangeSlices( } } - auto end_time = std::chrono::steady_clock::now(); - auto duration = std::chrono::duration_cast( - end_time - start_time); - // LOG(INFO) << "UpdateRangeSlices: Prepare slice batches and accumulate " - // "metadata for all ranges took " - // << duration.count() << "us" - // << ", req size = " << update_range_slice_reqs.size(); - // 2- Dispatch slice batches for all ranges concurrently (shared // SyncConcurrentRequest) SyncConcurrentRequest *slice_sync_concurrent = diff --git a/store_handler/data_store_service_client_closure.h b/store_handler/data_store_service_client_closure.h index 89449293..56469686 100644 --- a/store_handler/data_store_service_client_closure.h +++ b/store_handler/data_store_service_client_closure.h @@ -625,10 +625,8 @@ struct ReadBaseForArchiveCallbackData (*yield_fn)(); lk.lock(); waiting_.store(false, std::memory_order_release); - while (flying_read_cnt_ > 0) - { - cv_.wait(lk); - } + // If flying_read_cnt_ still > 0 after resume, loop and yield again. + // Do not use cv_.wait(): resume_fn path never notifies cv_. } } diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index b16d415f..b1e801b9 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -520,18 +520,18 @@ class LocalCcShards return (static_cast(allocated) >= range_slice_memory_limit_); #else - // table_ranges_heap_ usage commented out - use default heap - // if (table_ranges_heap_ != nullptr) - // { - // int64_t allocated, committed; - // mi_thread_stats(&allocated, &committed); - // return (static_cast(allocated) >= - // range_slice_memory_limit_); - // } - // else - // { - // return false; - // } + if (table_ranges_heap_ != nullptr) + { + int64_t allocated, committed; + mi_thread_stats(&allocated, &committed); + return (static_cast(allocated) >= + range_slice_memory_limit_); + } + else + { + return false; + } + return false; #endif } @@ -553,19 +553,17 @@ class LocalCcShards return (static_cast(allocated) <= target_memory_size); #else - // table_ranges_heap_ usage commented out - use default heap - // if (table_ranges_heap_ != nullptr) - // { - // size_t target_memory_size = range_slice_memory_limit_ / 10 * 9; - // int64_t allocated, committed; - // mi_thread_stats(&allocated, &committed); - // return (static_cast(allocated) <= target_memory_size); - // } - // else - // { - // return false; - // } - return true; + if (table_ranges_heap_ != nullptr) + { + size_t target_memory_size = range_slice_memory_limit_ / 10 * 9; + int64_t allocated, committed; + mi_thread_stats(&allocated, &committed); + return (static_cast(allocated) <= target_memory_size); + } + else + { + return false; + } #endif } @@ -580,31 +578,30 @@ class LocalCcShards << (bool) (static_cast(allocated) >= range_slice_memory_limit_); #else - // table_ranges_heap_ usage commented out - use default heap - // std::unique_lock heap_lk(table_ranges_heap_mux_); - // bool is_override_thd = mi_is_override_thread(); - // mi_threadid_t prev_thd = - // mi_override_thread(GetTableRangesHeapThreadId()); - // mi_heap_t *prev_heap = mi_heap_set_default(GetTableRangesHeap()); - // - // int64_t allocated, committed; - // mi_thread_stats(&allocated, &committed); - // LOG(INFO) << "Table range memory report: allocated " << allocated - // << ", committed " << committed << ", full: " - // << (bool) (static_cast(allocated) >= - // range_slice_memory_limit_); - // - // mi_heap_set_default(prev_heap); - // if (is_override_thd) - // { - // mi_override_thread(prev_thd); - // } - // else - // { - // mi_restore_default_thread_id(); - // } - // heap_lk.unlock(); - LOG(INFO) << "Table range memory report: table_ranges_heap_ disabled"; + + std::unique_lock heap_lk(table_ranges_heap_mux_); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = + mi_override_thread(GetTableRangesHeapThreadId()); + mi_heap_t *prev_heap = mi_heap_set_default(GetTableRangesHeap()); + + int64_t allocated, committed; + mi_thread_stats(&allocated, &committed); + LOG(INFO) << "Table range memory report: allocated " << allocated + << ", committed " << committed << ", full: " + << (bool) (static_cast(allocated) >= + range_slice_memory_limit_); + + mi_heap_set_default(prev_heap); + if (is_override_thd) + { + mi_override_thread(prev_thd); + } + else + { + mi_restore_default_thread_id(); + } + heap_lk.unlock(); #endif } @@ -883,21 +880,20 @@ class LocalCcShards } { std::unique_lock heap_lk(table_ranges_heap_mux_); - // bool is_override_thd = mi_is_override_thread(); - // mi_threadid_t prev_thd = - // mi_override_thread(GetTableRangesHeapThreadId()); - // table_ranges_heap_ usage commented out - // mi_heap_t *prev_heap = mi_heap_set_default(GetTableRangesHeap()); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = + mi_override_thread(GetTableRangesHeapThreadId()); + mi_heap_t *prev_heap = mi_heap_set_default(GetTableRangesHeap()); bool range_slice_mem_full = TableRangesMemoryFull(); - // mi_heap_set_default(prev_heap); - // if (is_override_thd) - // { - // mi_override_thread(prev_thd); - // } - // else - // { - // mi_restore_default_thread_id(); - // } + mi_heap_set_default(prev_heap); + if (is_override_thd) + { + mi_override_thread(prev_thd); + } + else + { + mi_restore_default_thread_id(); + } heap_lk.unlock(); if (range_slice_mem_full) { @@ -946,10 +942,9 @@ class LocalCcShards GetRangeOwnerInternal(partition_id, ng_id)->BucketOwner(); std::unique_lock heap_lk(table_ranges_heap_mux_); - // bool is_override_thd = mi_is_override_thread(); - // mi_threadid_t prev_thd = mi_override_thread(table_ranges_thread_id_); - // table_ranges_heap_ usage commented out - // mi_heap_t *prev_heap = mi_heap_set_default(table_ranges_heap_); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = mi_override_thread(table_ranges_thread_id_); + mi_heap_t *prev_heap = mi_heap_set_default(table_ranges_heap_); #if defined(WITH_JEMALLOC) uint32_t prev_arena_id; @@ -1012,8 +1007,8 @@ class LocalCcShards has_dml_since_ddl); } - // mi_heap_set_default(prev_heap); - /* + mi_heap_set_default(prev_heap); + if (is_override_thd) { mi_override_thread(prev_thd); @@ -1022,7 +1017,6 @@ class LocalCcShards { mi_restore_default_thread_id(); } - */ #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena_id); @@ -1062,9 +1056,8 @@ class LocalCcShards range_entry->UpdateRangeEntry(version, std::move(range_slices)); } - // table_ranges_heap_ usage commented out - // mi_heap_set_default(prev_heap); - /* + mi_heap_set_default(prev_heap); + if (is_override_thd) { mi_override_thread(prev_thd); @@ -1073,7 +1066,6 @@ class LocalCcShards { mi_restore_default_thread_id(); } - */ #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena_id); @@ -1932,6 +1924,7 @@ class LocalCcShards } void TimerRun(); + void TableRangesHeapThreadRun(); // Internal interface that exposes non const return type and does // not acquire mutex lock. TableRangeEntry *GetTableRangeEntryInternal( @@ -2039,6 +2032,14 @@ class LocalCcShards std::condition_variable timer_terminate_cv_; // std::atomic timer_terminate_; + // Background thread that initializes table_ranges_heap. After init, it + // sleeps until Shutdown. + std::thread table_ranges_heap_thd_; + bool table_ranges_heap_ready_{false}; + bool table_ranges_heap_terminate_{false}; + std::mutex table_ranges_heap_thd_mux_; + std::condition_variable table_ranges_heap_thd_cv_; + // When ccshard is full and no ccentry can be kicked-out, it will notify // checkpointer to do checkpoint and set flag is_wait_ckpt_ to true. // Subsequent ccrequest is able to skip checking freeable ccentry when diff --git a/tx_service/include/cc/range_cc_map.h b/tx_service/include/cc/range_cc_map.h index 28f00951..3a518093 100644 --- a/tx_service/include/cc/range_cc_map.h +++ b/tx_service/include/cc/range_cc_map.h @@ -1131,11 +1131,11 @@ class RangeCcMap : public TemplateCcMap std::unique_lock heap_lk( shard_->local_shards_.table_ranges_heap_mux_); - // bool is_override_thd = mi_is_override_thread(); - // mi_threadid_t prev_thd = mi_override_thread( - // shard_->local_shards_.GetTableRangesHeapThreadId()); - // mi_heap_t *prev_heap = mi_heap_set_default( - // shard_->local_shards_.GetTableRangesHeap()); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = mi_override_thread( + shard_->local_shards_.GetTableRangesHeapThreadId()); + mi_heap_t *prev_heap = mi_heap_set_default( + shard_->local_shards_.GetTableRangesHeap()); #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -1150,15 +1150,15 @@ class RangeCcMap : public TemplateCcMap this->cc_ng_id_, this->table_name_.IsBase()); - // mi_heap_set_default(prev_heap); - // if (is_override_thd) - // { - // mi_override_thread(prev_thd); - // } - // else - // { - // mi_restore_default_thread_id(); - // } + mi_heap_set_default(prev_heap); + if (is_override_thd) + { + mi_override_thread(prev_thd); + } + else + { + mi_restore_default_thread_id(); + } #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena); diff --git a/tx_service/src/cc/cc_req_misc.cpp b/tx_service/src/cc/cc_req_misc.cpp index db5e88df..2d6dbf31 100644 --- a/tx_service/src/cc/cc_req_misc.cpp +++ b/tx_service/src/cc/cc_req_misc.cpp @@ -408,11 +408,11 @@ void FetchRangeSlicesReq::SetFinish(CcErrorCode err) assert(range_entry_->RangeSlices() == nullptr); std::unique_lock heap_lk(shards->table_ranges_heap_mux_); - // bool is_override_thd = mi_is_override_thread(); - // mi_threadid_t prev_thd = - // mi_override_thread(shards->GetTableRangesHeapThreadId()); - // mi_heap_t *prev_heap = - // mi_heap_set_default(shards->GetTableRangesHeap()); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = + mi_override_thread(shards->GetTableRangesHeapThreadId()); + mi_heap_t *prev_heap = + mi_heap_set_default(shards->GetTableRangesHeap()); #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -429,15 +429,15 @@ void FetchRangeSlicesReq::SetFinish(CcErrorCode err) estimate_rec_size); bool range_slice_mem_full = shards->TableRangesMemoryFull(); - // mi_heap_set_default(prev_heap); - // if (is_override_thd) - // { - // mi_override_thread(prev_thd); - // } - // else - // { - // mi_restore_default_thread_id(); - // } + mi_heap_set_default(prev_heap); + if (is_override_thd) + { + mi_override_thread(prev_thd); + } + else + { + mi_restore_default_thread_id(); + } #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena); diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 30183482..0a34028f 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -178,8 +178,18 @@ LocalCcShards::LocalCcShards( timer_thd_ = std::thread([this] { TimerRun(); }); pthread_setname_np(timer_thd_.native_handle(), "tx_timer"); - // For mariadb, this thread is the main thread of the mariadb process. - InitializeTableRangesHeap(); + // Start background thread to initialize table_ranges_heap. For mariadb, the + // main thread is the mariadb process thread; offload heap init to avoid + // blocking it. + table_ranges_heap_thd_ = + std::thread([this] { TableRangesHeapThreadRun(); }); + pthread_setname_np(table_ranges_heap_thd_.native_handle(), + "table_ranges_heap"); + { + std::unique_lock lk(table_ranges_heap_thd_mux_); + table_ranges_heap_thd_cv_.wait( + lk, [this] { return table_ranges_heap_ready_; }); + } InitializeHashPartitionCkptHeap(); @@ -281,6 +291,13 @@ LocalCcShards::~LocalCcShards() } timer_thd_.join(); cc_shards_.clear(); + + { + std::scoped_lock lk(table_ranges_heap_thd_mux_); + table_ranges_heap_terminate_ = true; + table_ranges_heap_thd_cv_.notify_one(); + } + table_ranges_heap_thd_.join(); } void LocalCcShards::StartBackgroudWorkers() @@ -416,6 +433,19 @@ void LocalCcShards::BindThreadToFastMetaDataShard(size_t shard_idx) } } +void LocalCcShards::TableRangesHeapThreadRun() +{ + InitializeTableRangesHeap(); + { + std::lock_guard lk(table_ranges_heap_thd_mux_); + table_ranges_heap_ready_ = true; + table_ranges_heap_thd_cv_.notify_one(); + } + std::unique_lock lk(table_ranges_heap_thd_mux_); + table_ranges_heap_thd_cv_.wait( + lk, [this] { return table_ranges_heap_terminate_; }); +} + void LocalCcShards::TimerRun() { std::unique_lock lk(timer_terminate_mux_); @@ -1064,9 +1094,9 @@ void LocalCcShards::InitTableRanges(const TableName &range_table_name, std::unique_lock lk(fast_meta_data_mux_); std::unique_lock heap_lk(table_ranges_heap_mux_); - // bool is_override_thd = mi_is_override_thread(); - // mi_threadid_t prev_thd = mi_override_thread(table_ranges_thread_id_); - // mi_heap_t *prev_heap = mi_heap_set_default(table_ranges_heap_); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = mi_override_thread(table_ranges_thread_id_); + mi_heap_t *prev_heap = mi_heap_set_default(table_ranges_heap_); #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -1159,15 +1189,15 @@ void LocalCcShards::InitTableRanges(const TableName &range_table_name, } } - // mi_heap_set_default(prev_heap); - // if (is_override_thd) - // { - // mi_override_thread(prev_thd); - // } - // else - // { - // mi_restore_default_thread_id(); - // } + mi_heap_set_default(prev_heap); + if (is_override_thd) + { + mi_override_thread(prev_thd); + } + else + { + mi_restore_default_thread_id(); + } #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena); @@ -1337,11 +1367,11 @@ void LocalCcShards::KickoutRangeSlices() { std::unique_lock heap_lk( table_ranges_heap_mux_); - // bool is_override_thd = mi_is_override_thread(); - // mi_threadid_t prev_thd = mi_override_thread( - // GetTableRangesHeapThreadId()); - // mi_heap_t *prev_heap = - // mi_heap_set_default(GetTableRangesHeap()); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = mi_override_thread( + GetTableRangesHeapThreadId()); + mi_heap_t *prev_heap = + mi_heap_set_default(GetTableRangesHeap()); #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -1355,15 +1385,15 @@ void LocalCcShards::KickoutRangeSlices() range_entry->DropStoreRange(); bool has_enough_mem = HasEnoughTableRangesMemory(); - // mi_heap_set_default(prev_heap); - // if (is_override_thd) - // { - // mi_override_thread(prev_thd); - // } - // else - // { - // mi_restore_default_thread_id(); - // } + mi_heap_set_default(prev_heap); + if (is_override_thd) + { + mi_override_thread(prev_thd); + } + else + { + mi_restore_default_thread_id(); + } #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena); @@ -1401,12 +1431,10 @@ void LocalCcShards::KickoutRangeSlices() if (entry->IsStoreRangeFree()) { std::unique_lock heap_lk(table_ranges_heap_mux_); - /* bool is_override_thd = mi_is_override_thread(); mi_threadid_t prev_thd = mi_override_thread(GetTableRangesHeapThreadId()); mi_heap_t *prev_heap = mi_heap_set_default(GetTableRangesHeap()); - */ #if defined(WITH_JEMALLOC) uint32_t prev_arena; JemallocArenaSwitcher::ReadCurrentArena(prev_arena); @@ -1418,7 +1446,6 @@ void LocalCcShards::KickoutRangeSlices() entry->DropStoreRange(); bool has_enough_mem = HasEnoughTableRangesMemory(); - /* mi_heap_set_default(prev_heap); if (is_override_thd) { @@ -1428,7 +1455,6 @@ void LocalCcShards::KickoutRangeSlices() { mi_restore_default_thread_id(); } - */ #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena); @@ -2148,13 +2174,11 @@ bool LocalCcShards::DropStoreRangesInBucket(NodeGroupId ng_id, { std::unique_lock heap_lk( table_ranges_heap_mux_); - /* bool is_override_thd = mi_is_override_thread(); mi_threadid_t prev_thd = mi_override_thread(GetTableRangesHeapThreadId()); mi_heap_t *prev_heap = mi_heap_set_default(GetTableRangesHeap()); - */ #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -2167,7 +2191,6 @@ bool LocalCcShards::DropStoreRangesInBucket(NodeGroupId ng_id, entry->DropStoreRange(); - /* mi_heap_set_default(prev_heap); if (is_override_thd) { @@ -2177,7 +2200,6 @@ bool LocalCcShards::DropStoreRangesInBucket(NodeGroupId ng_id, { mi_restore_default_thread_id(); } - */ #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena); diff --git a/tx_service/src/cc/range_slice.cpp b/tx_service/src/cc/range_slice.cpp index 03eda6b1..68cfa4b0 100644 --- a/tx_service/src/cc/range_slice.cpp +++ b/tx_service/src/cc/range_slice.cpp @@ -510,11 +510,11 @@ void StoreRange::UpdateSliceSpec(StoreSlice *slice, std::unique_lock heap_lk( local_cc_shards_.table_ranges_heap_mux_); - // bool is_override_thd = mi_is_override_thread(); - // mi_threadid_t prev_thd = - // mi_override_thread(local_cc_shards_.GetTableRangesHeapThreadId()); - // mi_heap_t *prev_heap = - // mi_heap_set_default(local_cc_shards_.GetTableRangesHeap()); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = + mi_override_thread(local_cc_shards_.GetTableRangesHeapThreadId()); + mi_heap_t *prev_heap = + mi_heap_set_default(local_cc_shards_.GetTableRangesHeap()); #if defined(WITH_JEMALLOC) uint32_t prev_arena; @@ -527,15 +527,15 @@ void StoreRange::UpdateSliceSpec(StoreSlice *slice, UpdateSlice(slice, split_keys); bool range_slice_mem_full = local_cc_shards_.TableRangesMemoryFull(); - // mi_heap_set_default(prev_heap); - // if (is_override_thd) - // { - // mi_override_thread(prev_thd); - // } - // else - // { - // mi_restore_default_thread_id(); - // } + mi_heap_set_default(prev_heap); + if (is_override_thd) + { + mi_override_thread(prev_thd); + } + else + { + mi_restore_default_thread_id(); + } #if defined(WITH_JEMALLOC) JemallocArenaSwitcher::SwitchToArena(prev_arena); #endif From b284ed6e79f30fedd7e15cce186e6829a04d1096 Mon Sep 17 00:00:00 2001 From: lokax Date: Wed, 18 Mar 2026 18:38:02 +0800 Subject: [PATCH 34/38] debug log --- store_handler/data_store_service_client.cpp | 3 +++ store_handler/data_store_service_client_closure.h | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index 3bd9f0fe..bbadce22 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -3269,6 +3269,7 @@ bool DataStoreServiceClient::CopyBaseToArchiveImpl( { if (yield_fptr && resume_fptr) { + LOG(INFO) << "== wait: this = " << callback_data; callback_data->Wait(yield_fptr, resume_fptr); } else @@ -3287,10 +3288,12 @@ bool DataStoreServiceClient::CopyBaseToArchiveImpl( // invalid memory in callback. if (yield_fptr && resume_fptr) { + LOG(INFO) << "== wait: this = " << &callback_datas[0]; callback_datas[0].Wait(yield_fptr, resume_fptr); } else { + LOG(INFO) << "== wait: this = " << &callback_datas[0]; std::unique_lock lk(mtx); while (flying_cnt > 0) { diff --git a/store_handler/data_store_service_client_closure.h b/store_handler/data_store_service_client_closure.h index 56469686..e8f8e9c6 100644 --- a/store_handler/data_store_service_client_closure.h +++ b/store_handler/data_store_service_client_closure.h @@ -622,6 +622,8 @@ struct ReadBaseForArchiveCallbackData { waiting_.store(true, std::memory_order_release); lk.unlock(); + LOG(INFO) << "== wait: this = " << this + << ", flying read cnt = " << flying_read_cnt_; (*yield_fn)(); lk.lock(); waiting_.store(false, std::memory_order_release); @@ -641,6 +643,8 @@ struct ReadBaseForArchiveCallbackData { std::unique_lock lk(mtx_); flying_read_cnt_--; + LOG(INFO) << "== decrease flying read cnt: this = " << this + << ", flying read cnt = " << flying_read_cnt_; if (flying_read_cnt_ == 0) { if (resume_fn_ && waiting_.load(std::memory_order_acquire)) @@ -648,6 +652,7 @@ struct ReadBaseForArchiveCallbackData waiting_.store(false, std::memory_order_release); auto *fn = resume_fn_; lk.unlock(); + LOG(INFO) << "== notify, this = " << this; (*fn)(); // Do not re-lock: resume_fn may acquire flush_worker_mux. // Do not access members after fn(): object may be destroyed. From 511c84d741a7bff24ea04cff4e25f18935b2b6a2 Mon Sep 17 00:00:00 2001 From: lokax Date: Thu, 19 Mar 2026 10:56:43 +0800 Subject: [PATCH 35/38] fix --- store_handler/data_store_service_client.cpp | 3 ++- store_handler/data_store_service_client_closure.h | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index bbadce22..e41a8890 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -3233,6 +3233,7 @@ bool DataStoreServiceClient::CopyBaseToArchiveImpl( bthread::ConditionVariable cv; size_t flying_cnt = 0; int error_code = 0; + std::atomic waiting{false}; // shared: last callback must notify Wait // Use deque: ReadBaseForArchiveCallbackData contains std::atomic // which is non-copyable/non-movable; vector requires element // move/copy on realloc, deque does not. @@ -3240,7 +3241,7 @@ bool DataStoreServiceClient::CopyBaseToArchiveImpl( for (size_t i = 0; i < base_vec.size(); ++i) { callback_datas.emplace_back( - mtx, cv, flying_cnt, error_code, resume_fptr); + mtx, cv, flying_cnt, error_code, waiting, resume_fptr); } for (size_t base_idx = 0; base_idx < base_vec.size(); ++base_idx) diff --git a/store_handler/data_store_service_client_closure.h b/store_handler/data_store_service_client_closure.h index e8f8e9c6..4d48bba0 100644 --- a/store_handler/data_store_service_client_closure.h +++ b/store_handler/data_store_service_client_closure.h @@ -571,11 +571,13 @@ struct ReadBaseForArchiveCallbackData bthread::ConditionVariable &cv, size_t &flying_read_cnt, int &error_code, + std::atomic &waiting, const std::function *resume_fn = nullptr) : mtx_(mtx), cv_(cv), flying_read_cnt_(flying_read_cnt), error_code_(error_code), + waiting_(waiting), resume_fn_(resume_fn), partition_id_(0), key_str_(), @@ -705,8 +707,8 @@ struct ReadBaseForArchiveCallbackData bthread::ConditionVariable &cv_; size_t &flying_read_cnt_; int &error_code_; + std::atomic &waiting_; // shared: any callback's DecreaseFlyingReadCount can notify const std::function *resume_fn_{nullptr}; - std::atomic waiting_{false}; int32_t partition_id_; std::string_view key_str_; std::string value_str_; From 2a9805795376c58e890b754b1dc540ace7e61229 Mon Sep 17 00:00:00 2001 From: lokax Date: Thu, 19 Mar 2026 16:24:10 +0800 Subject: [PATCH 36/38] fix hash ckpt heap --- tx_service/include/cc/local_cc_shards.h | 18 ++++++++++-- tx_service/src/cc/local_cc_shards.cpp | 37 +++++++++++++++++++++---- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index b1e801b9..d014f6ab 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -474,8 +474,13 @@ class LocalCcShards void InitializeHashPartitionCkptHeap() { - hash_partition_ckpt_heap_ = mi_heap_new(); - hash_partition_main_thread_id_ = mi_thread_id(); + std::unique_lock lk(hash_partition_ckpt_heap_mux_); + if (!hash_partition_ckpt_heap_) + { + hash_partition_main_thread_id_ = mi_thread_id(); + hash_partition_ckpt_heap_ = mi_heap_new(); + } + #if defined(WITH_JEMALLOC) // create hash partition ckpt arena size_t sz = sizeof(uint32_t); @@ -1925,6 +1930,7 @@ class LocalCcShards void TimerRun(); void TableRangesHeapThreadRun(); + void HashPartitionCkptHeapThreadRun(); // Internal interface that exposes non const return type and does // not acquire mutex lock. TableRangeEntry *GetTableRangeEntryInternal( @@ -2040,6 +2046,14 @@ class LocalCcShards std::mutex table_ranges_heap_thd_mux_; std::condition_variable table_ranges_heap_thd_cv_; + // Background thread that initializes hash_partition_ckpt_heap. After init, + // it sleeps until Shutdown. + std::thread hash_partition_ckpt_heap_thd_; + bool hash_partition_ckpt_heap_ready_{false}; + bool hash_partition_ckpt_heap_terminate_{false}; + std::mutex hash_partition_ckpt_heap_thd_mux_; + std::condition_variable hash_partition_ckpt_heap_thd_cv_; + // When ccshard is full and no ccentry can be kicked-out, it will notify // checkpointer to do checkpoint and set flag is_wait_ckpt_ to true. // Subsequent ccrequest is able to skip checking freeable ccentry when diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 0a34028f..4111c05e 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -191,7 +191,16 @@ LocalCcShards::LocalCcShards( lk, [this] { return table_ranges_heap_ready_; }); } - InitializeHashPartitionCkptHeap(); + // Start background thread to initialize hash_partition_ckpt_heap. + hash_partition_ckpt_heap_thd_ = + std::thread([this] { HashPartitionCkptHeapThreadRun(); }); + pthread_setname_np(hash_partition_ckpt_heap_thd_.native_handle(), + "hash_part_ckpt_heap"); + { + std::unique_lock lk(hash_partition_ckpt_heap_thd_mux_); + hash_partition_ckpt_heap_thd_cv_.wait( + lk, [this] { return hash_partition_ckpt_heap_ready_; }); + } std::set ng_ids; for (auto &[ng_id, _] : *ng_configs) @@ -292,6 +301,13 @@ LocalCcShards::~LocalCcShards() timer_thd_.join(); cc_shards_.clear(); + { + std::scoped_lock lk(hash_partition_ckpt_heap_thd_mux_); + hash_partition_ckpt_heap_terminate_ = true; + hash_partition_ckpt_heap_thd_cv_.notify_one(); + } + hash_partition_ckpt_heap_thd_.join(); + { std::scoped_lock lk(table_ranges_heap_thd_mux_); table_ranges_heap_terminate_ = true; @@ -436,16 +452,25 @@ void LocalCcShards::BindThreadToFastMetaDataShard(size_t shard_idx) void LocalCcShards::TableRangesHeapThreadRun() { InitializeTableRangesHeap(); - { - std::lock_guard lk(table_ranges_heap_thd_mux_); - table_ranges_heap_ready_ = true; - table_ranges_heap_thd_cv_.notify_one(); - } + std::unique_lock lk(table_ranges_heap_thd_mux_); + table_ranges_heap_ready_ = true; + table_ranges_heap_thd_cv_.notify_one(); table_ranges_heap_thd_cv_.wait( lk, [this] { return table_ranges_heap_terminate_; }); } +void LocalCcShards::HashPartitionCkptHeapThreadRun() +{ + InitializeHashPartitionCkptHeap(); + + std::unique_lock lk(hash_partition_ckpt_heap_thd_mux_); + hash_partition_ckpt_heap_ready_ = true; + hash_partition_ckpt_heap_thd_cv_.notify_one(); + hash_partition_ckpt_heap_thd_cv_.wait( + lk, [this] { return hash_partition_ckpt_heap_terminate_; }); +} + void LocalCcShards::TimerRun() { std::unique_lock lk(timer_terminate_mux_); From 616d1ab3d6254b57a0bde346b8af9cc265fe52df Mon Sep 17 00:00:00 2001 From: lokax Date: Thu, 19 Mar 2026 17:28:02 +0800 Subject: [PATCH 37/38] remove log --- store_handler/data_store_service_client.cpp | 6 ++---- .../data_store_service_client_closure.h | 8 ++------ tx_service/include/cc/catalog_cc_map.h | 8 -------- tx_service/include/cc/range_slice.h | 13 ------------- tx_service/include/range_record.h | 16 ---------------- 5 files changed, 4 insertions(+), 47 deletions(-) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index e41a8890..26e1d366 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -3233,7 +3233,8 @@ bool DataStoreServiceClient::CopyBaseToArchiveImpl( bthread::ConditionVariable cv; size_t flying_cnt = 0; int error_code = 0; - std::atomic waiting{false}; // shared: last callback must notify Wait + std::atomic waiting{ + false}; // shared: last callback must notify Wait // Use deque: ReadBaseForArchiveCallbackData contains std::atomic // which is non-copyable/non-movable; vector requires element // move/copy on realloc, deque does not. @@ -3270,7 +3271,6 @@ bool DataStoreServiceClient::CopyBaseToArchiveImpl( { if (yield_fptr && resume_fptr) { - LOG(INFO) << "== wait: this = " << callback_data; callback_data->Wait(yield_fptr, resume_fptr); } else @@ -3289,12 +3289,10 @@ bool DataStoreServiceClient::CopyBaseToArchiveImpl( // invalid memory in callback. if (yield_fptr && resume_fptr) { - LOG(INFO) << "== wait: this = " << &callback_datas[0]; callback_datas[0].Wait(yield_fptr, resume_fptr); } else { - LOG(INFO) << "== wait: this = " << &callback_datas[0]; std::unique_lock lk(mtx); while (flying_cnt > 0) { diff --git a/store_handler/data_store_service_client_closure.h b/store_handler/data_store_service_client_closure.h index 4d48bba0..00de7d0a 100644 --- a/store_handler/data_store_service_client_closure.h +++ b/store_handler/data_store_service_client_closure.h @@ -624,8 +624,6 @@ struct ReadBaseForArchiveCallbackData { waiting_.store(true, std::memory_order_release); lk.unlock(); - LOG(INFO) << "== wait: this = " << this - << ", flying read cnt = " << flying_read_cnt_; (*yield_fn)(); lk.lock(); waiting_.store(false, std::memory_order_release); @@ -645,8 +643,6 @@ struct ReadBaseForArchiveCallbackData { std::unique_lock lk(mtx_); flying_read_cnt_--; - LOG(INFO) << "== decrease flying read cnt: this = " << this - << ", flying read cnt = " << flying_read_cnt_; if (flying_read_cnt_ == 0) { if (resume_fn_ && waiting_.load(std::memory_order_acquire)) @@ -654,7 +650,6 @@ struct ReadBaseForArchiveCallbackData waiting_.store(false, std::memory_order_release); auto *fn = resume_fn_; lk.unlock(); - LOG(INFO) << "== notify, this = " << this; (*fn)(); // Do not re-lock: resume_fn may acquire flush_worker_mux. // Do not access members after fn(): object may be destroyed. @@ -707,7 +702,8 @@ struct ReadBaseForArchiveCallbackData bthread::ConditionVariable &cv_; size_t &flying_read_cnt_; int &error_code_; - std::atomic &waiting_; // shared: any callback's DecreaseFlyingReadCount can notify + std::atomic + &waiting_; // shared: any callback's DecreaseFlyingReadCount can notify const std::function *resume_fn_{nullptr}; int32_t partition_id_; std::string_view key_str_; diff --git a/tx_service/include/cc/catalog_cc_map.h b/tx_service/include/cc/catalog_cc_map.h index 3f995d06..69613ec1 100644 --- a/tx_service/include/cc/catalog_cc_map.h +++ b/tx_service/include/cc/catalog_cc_map.h @@ -463,14 +463,6 @@ class CatalogCcMap UINT64_MAX, false); - LOG(INFO) - << "catalog init range slices: addr=" - << static_cast( - range.second->RangeSlices()) - << " partition_id=" - << range.second->RangeSlices() - ->PartitionId(); - mi_heap_set_default(prev_heap); if (is_override_thd) { diff --git a/tx_service/include/cc/range_slice.h b/tx_service/include/cc/range_slice.h index 75711b4e..2f74ecd2 100644 --- a/tx_service/include/cc/range_slice.h +++ b/tx_service/include/cc/range_slice.h @@ -888,23 +888,10 @@ class TemplateStoreRange : public StoreRange init_key_cache, empty_range ? init_key_cache : false); slices_.emplace_back(std::move(slice)); - if (partition_id_ == 0 && partition_id != 0) - { - LOG(INFO) << "StoreRange init: addr=" << static_cast(this) - << " partition_id=" << partition_id_ - << ", part id = " << partition_id; - assert(false); - } - LOG(INFO) << "StoreRange init: addr=" << static_cast(this) - << " partition_id=" << partition_id_ - << ", part id = " << partition_id; } ~TemplateStoreRange() { - LOG(INFO) << "StoreRange destructor: addr=" - << static_cast(this); - // << " partition_id=" << partition_id_; } void SetRangeEndKey(const KeyT *end_key) diff --git a/tx_service/include/range_record.h b/tx_service/include/range_record.h index cb3d11a8..4ec03f47 100644 --- a/tx_service/include/range_record.h +++ b/tx_service/include/range_record.h @@ -546,9 +546,6 @@ struct TemplateTableRangeEntry : public TableRangeEntry ~TemplateTableRangeEntry() { - // LOG(INFO) << "TemplateTableRangeEntry destructor: addr=" - // << static_cast(this) - // << " partition_id=" << range_info_.PartitionId(); } void UpdateRangeEntry(uint64_t version_ts, @@ -606,18 +603,8 @@ struct TemplateTableRangeEntry : public TableRangeEntry if (!empty_range) { range_slices->InitSlices(std::move(slices)); - LOG(INFO) << "TemplateTableRangeEntry InitRangeSlices: addr=" - << static_cast(range_slices.get()) - << " partition_id=" << range_slices->PartitionId() - << ", start key = " - << range_slices->RangeStartKey()->ToString(); } range_slices_ = std::move(range_slices); - LOG(INFO) << "TemplateTableRangeEntry InitRangeSlices: addr=" - << static_cast(range_slices_.get()) - << " partition_id=" << range_slices_->PartitionId() - << ", start key = " - << range_slices_->RangeStartKey()->ToString(); } /** @@ -653,9 +640,6 @@ struct TemplateTableRangeEntry : public TableRangeEntry if (range_slices_) { assert(range_slices_->Pins() == 0); - LOG(INFO) << "StoreRange DropStoreRange: addr=" - << static_cast(range_slices_.get()) - << " partition_id=" << range_slices_->PartitionId(); range_slices_ = nullptr; } } From 21b539c9b7cda760127749bdbd1f15908368c7cc Mon Sep 17 00:00:00 2001 From: lokax Date: Mon, 23 Mar 2026 11:47:04 +0800 Subject: [PATCH 38/38] remove debug log --- store_handler/data_store_service_client.cpp | 12 ---- store_handler/data_store_service_client.h | 71 +++++++++---------- .../data_store_service_client_closure.cpp | 20 ------ store_handler/rocksdb_handler.h | 39 +++++----- tx_service/include/cc/cc_req_misc.h | 6 -- tx_service/include/cc/local_cc_shards.h | 7 +- tx_service/include/cc/range_slice.h | 4 +- tx_service/include/fault/log_replay_service.h | 5 +- tx_service/include/range_record.h | 6 +- tx_service/include/store/int_mem_store.h | 37 +++++----- tx_service/src/cc/local_cc_shards.cpp | 66 +---------------- tx_service/src/fault/log_replay_service.cpp | 7 +- tx_service/src/tx_index_operation.cpp | 34 +++++---- 13 files changed, 101 insertions(+), 213 deletions(-) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index 26e1d366..1882b79d 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -500,9 +500,7 @@ bool DataStoreServiceClient::PutAllImpl( // Wait for all partitions to complete if (yield_fptr != nullptr && resume_fptr != nullptr) { - // LOG(INFO) << "PutAllImpl: Before Wait, this = " << sync_putall; sync_putall->Wait(yield_fptr, resume_fptr); - // LOG(INFO) << "PutAllImpl: After Wait, this = " << sync_putall; } else { @@ -1848,9 +1846,7 @@ bool DataStoreServiceClient::UpdateRangeSlices( ++iterations_since_yield; if (iterations_since_yield >= MAX_ITERATIONS_WITHOUT_YIELD) { - // LOG(INFO) << "UpdateRangeSlices: prepare Before sync yield"; (*sync_yield_fptr)(); - // LOG(INFO) << "UpdateRangeSlices: prepare After sync yield"; iterations_since_yield = 0; } } @@ -1880,18 +1876,14 @@ bool DataStoreServiceClient::UpdateRangeSlices( ++iterations_since_yield; if (iterations_since_yield >= MAX_ITERATIONS_WITHOUT_YIELD) { - // LOG(INFO) << "UpdateRangeSlices: Before sync yield"; (*sync_yield_fptr)(); - // LOG(INFO) << "UpdateRangeSlices: After sync yield"; iterations_since_yield = 0; } } } - // LOG(INFO) << "UpdateRangeSlices: before WaitAll slice sync"; // 3- Wait for slice requests to complete slice_sync_concurrent->WaitForAll(); - // LOG(INFO) << "UpdateRangeSlices: after WaitAll slice sync"; if (slice_sync_concurrent->result_.error_code() != remote::DataStoreError::NO_ERROR) @@ -1946,10 +1938,8 @@ bool DataStoreServiceClient::UpdateRangeSlices( DispatchRangeMetadataBatches( kv_range_table_name, meta_acc, meta_sync_concurrent); - // LOG(INFO) << "UpdateRangeSlices: before WaitAll meta sync"; // 5- Wait for metadata requests to complete meta_sync_concurrent->WaitForAll(); - // LOG(INFO) << "UpdateRangeSlices: after WaitAll meta sync"; // 6- Check for errors if (meta_sync_concurrent->result_.error_code() != @@ -1975,9 +1965,7 @@ bool DataStoreServiceClient::UpdateRangeSlices( FlushData(kv_range_table_names, callback_data, &SyncCallback); if (yield_fptr != nullptr && resume_fptr != nullptr) { - // LOG(INFO) << "UpdateRangeSlices: before Wait meta sync"; callback_data->Wait(yield_fptr, resume_fptr); - // LOG(INFO) << "UpdateRangeSlices: after Wait meta sync"; } else { diff --git a/store_handler/data_store_service_client.h b/store_handler/data_store_service_client.h index f77425fe..b3b02792 100644 --- a/store_handler/data_store_service_client.h +++ b/store_handler/data_store_service_client.h @@ -229,14 +229,14 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler * @param node_group * @return whether all entries are written to data store successfully */ - bool PutAll(std::unordered_map< - std::string_view, - std::vector>> - &flush_task, - const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr, - const std::function *sync_yield_fptr = nullptr) - override; + bool PutAll( + std::unordered_map< + std::string_view, + std::vector>> + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr, + const std::function *sync_yield_fptr = nullptr) override; bool NeedPersistKV() override { @@ -342,12 +342,12 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler uint32_t range_partition_id, txservice::FillStoreSliceCc *load_slice_req) override; - bool UpdateRangeSlices(const std::vector - &update_range_slice_reqs, - const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr, - const std::function *sync_yield_fptr = nullptr) - override; + bool UpdateRangeSlices( + const std::vector + &update_range_slice_reqs, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr, + const std::function *sync_yield_fptr = nullptr) override; bool UpdateRangeSlices(const txservice::TableName &table_name, uint64_t version, @@ -433,13 +433,13 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler * @brief Write batch historical versions into DataStore. * */ - bool PutArchivesAll(std::unordered_map< - std::string_view, - std::vector>> - &flush_task, - const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr) - override; + bool PutArchivesAll( + std::unordered_map< + std::string_view, + std::vector>> + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) override; /** * @brief Copy record from base/sk table to mvcc_archives. @@ -614,13 +614,13 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler bool is_range_partition) const; private: - bool PutArchivesAllImpl(std::unordered_map< - std::string_view, - std::vector< - std::unique_ptr>> - &flush_task, - const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr); + bool PutArchivesAllImpl( + std::unordered_map< + std::string_view, + std::vector>> + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr); bool PutAllImpl(std::unordered_map< std::string_view, @@ -630,14 +630,13 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler const std::function *resume_fptr = nullptr, const std::function *sync_yield_fptr = nullptr); - bool CopyBaseToArchiveImpl(std::unordered_map< - std::string_view, - std::vector< - std::unique_ptr< - txservice::FlushTaskEntry>>> - &flush_task, - const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr); + bool CopyBaseToArchiveImpl( + std::unordered_map< + std::string_view, + std::vector>> + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr); int32_t MapKeyHashToPartitionId(const txservice::TxKey &key) const { diff --git a/store_handler/data_store_service_client_closure.cpp b/store_handler/data_store_service_client_closure.cpp index 13cd69de..950cabc6 100644 --- a/store_handler/data_store_service_client_closure.cpp +++ b/store_handler/data_store_service_client_closure.cpp @@ -535,9 +535,7 @@ void SyncPutAllData::Wait(const std::function *yield_fn, { waiting_.store(true, std::memory_order_release); lk.unlock(); - // LOG(INFO) << "SyncPutAllData Wait: Before yield, this = " << this; (*yield_fn)(); - // LOG(INFO) << "SynPutAllData Wait: After yield, this = " << this; lk.lock(); waiting_.store(false, std::memory_order_release); } @@ -552,17 +550,7 @@ void SyncConcurrentRequest::WaitForCapacityAndIncrement() { waiting_.store(true, std::memory_order_release); lk.unlock(); - // LOG(INFO) << "SyncConcurrentRequest WaitForCapacityAndIncrement: - // " - // "Before yield, this = " - // << this; - (*yield_fn_)(); - - // LOG(INFO) << "SyncConcurrentRequest WaitForCapacityAndIncrement: - // " - // "After yield, this = " - // << this; lk.lock(); waiting_.store(false, std::memory_order_release); } @@ -584,15 +572,7 @@ void SyncConcurrentRequest::WaitForAll() { waiting_.store(true, std::memory_order_release); lk.unlock(); - // LOG(INFO) - // << "SyncConcurrentRequest WaitForAll: Before yield, this = " - // << this; - (*yield_fn_)(); - - // LOG(INFO) - // << "SyncConcurrentRequest WaitForAll: After yield, this = " - // << this; lk.lock(); waiting_.store(false, std::memory_order_release); } diff --git a/store_handler/rocksdb_handler.h b/store_handler/rocksdb_handler.h index 72eb848d..af59eaac 100644 --- a/store_handler/rocksdb_handler.h +++ b/store_handler/rocksdb_handler.h @@ -272,13 +272,13 @@ class RocksDBHandler : public txservice::store::DataStoreHandler * @param node_group * @return whether all entries are written to data store successfully */ - bool PutAll(std::unordered_map< - std::string_view, - std::vector>> &batch, - const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr, - const std::function *sync_yield_fptr = nullptr) - override; + bool PutAll( + std::unordered_map< + std::string_view, + std::vector>> &batch, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr, + const std::function *sync_yield_fptr = nullptr) override; /** * @brief indicate end of flush entries in a single ckpt for \@param @@ -416,12 +416,12 @@ class RocksDBHandler : public txservice::store::DataStoreHandler int32_t partition_id, uint64_t range_version) override; - bool UpdateRangeSlices(const std::vector - &update_range_slice_reqs, - const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr, - const std::function *sync_yield_fptr = nullptr) - override; + bool UpdateRangeSlices( + const std::vector + &update_range_slice_reqs, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr, + const std::function *sync_yield_fptr = nullptr) override; bool UpsertRanges(const txservice::TableName &table_name, std::vector range_info, @@ -486,13 +486,12 @@ class RocksDBHandler : public txservice::store::DataStoreHandler * @brief Write batch historical versions into DataStore. * */ - bool PutArchivesAll(std::unordered_map< - std::string_view, - std::vector>> - &batch, - const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr) - override; + bool PutArchivesAll( + std::unordered_map< + std::string_view, + std::vector>> &batch, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) override; /** * @brief Copy record from base/sk table to mvcc_archives. */ diff --git a/tx_service/include/cc/cc_req_misc.h b/tx_service/include/cc/cc_req_misc.h index 9d8835d2..51f4d050 100644 --- a/tx_service/include/cc/cc_req_misc.h +++ b/tx_service/include/cc/cc_req_misc.h @@ -907,9 +907,7 @@ struct WaitableCc : public RunOnTxProcessorCc { waiting_.store(true, std::memory_order_release); lk.unlock(); - // LOG(INFO) << "WaitableCc Wait: Before yield"; (*yield_fn)(); - // LOG(INFO) << "WaitableCc Wait: After Yield"; lk.lock(); waiting_.store(false, std::memory_order_release); } @@ -969,7 +967,6 @@ struct WaitableCc : public RunOnTxProcessorCc waiting_.store(false, std::memory_order_release); auto *fn = resume_fn_; lk.unlock(); - // LOG(INFO) << "WaitableCc Execute: before resume"; (*fn)(); } else if (resume_fn_ == nullptr) @@ -1067,7 +1064,6 @@ struct UpdateCceCkptTsCc : public CcRequestBase waiting_.store(false, std::memory_order_release); auto *fn = resume_fn_; lk.unlock(); - // LOG(INFO) << "UpdateCceCkptTsCc SetFinished: before resume"; (*fn)(); } else if (resume_fn_ == nullptr) @@ -1099,9 +1095,7 @@ struct UpdateCceCkptTsCc : public CcRequestBase { waiting_.store(true, std::memory_order_release); lk.unlock(); - // LOG(INFO) << "UpdateCceCkptTsCc Wait: Before yield"; (*yield_fn)(); - // LOG(INFO) << "UpdateCceCkptTsCc Wait: After Yield"; lk.lock(); waiting_.store(false, std::memory_order_release); } diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index d014f6ab..c28229a9 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -92,7 +92,6 @@ struct CoroCtx { ~CoroCtx() { - // LOG(INFO) << "CoroCtx destructor, this = " << this; } boost::context::continuation coro_; @@ -536,8 +535,6 @@ class LocalCcShards { return false; } - - return false; #endif } @@ -2581,10 +2578,10 @@ class LocalCcShards #ifndef NDEBUG boost::context::protected_fixedsize_stack flush_coro_stack_allocator_{ - 512 * 1024}; // 512KB, guard page for stack overflow detection + 128 * 1024}; // 128KB, guard page for stack overflow detection #else boost::context::pooled_fixedsize_stack flush_coro_stack_allocator_{ - 1024 * 1024}; // 1MB for FlushData call chain + 128 * 1024}; // 128KB for FlushData call chain #endif void FlushDataWorker(size_t worker_idx); diff --git a/tx_service/include/cc/range_slice.h b/tx_service/include/cc/range_slice.h index 2f74ecd2..0961534c 100644 --- a/tx_service/include/cc/range_slice.h +++ b/tx_service/include/cc/range_slice.h @@ -890,9 +890,7 @@ class TemplateStoreRange : public StoreRange slices_.emplace_back(std::move(slice)); } - ~TemplateStoreRange() - { - } + ~TemplateStoreRange() = default; void SetRangeEndKey(const KeyT *end_key) { diff --git a/tx_service/include/fault/log_replay_service.h b/tx_service/include/fault/log_replay_service.h index fa63692f..8d69f06d 100644 --- a/tx_service/include/fault/log_replay_service.h +++ b/tx_service/include/fault/log_replay_service.h @@ -120,10 +120,7 @@ class RecoveryService : public brpc::StreamInputHandler, TxLog *log_agent, std::string ip, uint16_t port); - ~RecoveryService() - { - DLOG(INFO) << "RecoveryService destructor"; - } + ~RecoveryService() = default; void Shutdown(); diff --git a/tx_service/include/range_record.h b/tx_service/include/range_record.h index 4ec03f47..a7af99d6 100644 --- a/tx_service/include/range_record.h +++ b/tx_service/include/range_record.h @@ -32,8 +32,6 @@ #include #include -#include "glog/logging.h" - // #include "cc_req_misc.h" #include "data_sync_task.h" #include "range_bucket_key_record.h" @@ -544,9 +542,7 @@ struct TemplateTableRangeEntry : public TableRangeEntry { } - ~TemplateTableRangeEntry() - { - } + ~TemplateTableRangeEntry() = default; void UpdateRangeEntry(uint64_t version_ts, std::unique_ptr> slices) diff --git a/tx_service/include/store/int_mem_store.h b/tx_service/include/store/int_mem_store.h index 3ba32baf..39c12c8f 100644 --- a/tx_service/include/store/int_mem_store.h +++ b/tx_service/include/store/int_mem_store.h @@ -55,17 +55,16 @@ class IntMemoryStore : public DataStoreHandler * @return whether all entries are written to data store successfully */ bool PutAll(std::unordered_map< - std::string_view, - std::vector>> + std::string_view, + std::vector>> &flush_task, const std::function *yield_fptr = nullptr, const std::function *resume_fptr = nullptr, - const std::function *sync_yield_fptr = nullptr) - override + const std::function *sync_yield_fptr = nullptr) override { - (void)yield_fptr; - (void)resume_fptr; - (void)sync_yield_fptr; + (void) yield_fptr; + (void) resume_fptr; + (void) sync_yield_fptr; assert(false); // for (const auto &ref : batch) // { @@ -250,16 +249,16 @@ class IntMemoryStore : public DataStoreHandler * @brief Write batch historical versions into DataStore. * */ - bool PutArchivesAll(std::unordered_map< - std::string_view, - std::vector>> - &flush_task, - const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr) - override - { - (void)yield_fptr; - (void)resume_fptr; + bool PutArchivesAll( + std::unordered_map< + std::string_view, + std::vector>> + &flush_task, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr) override + { + (void) yield_fptr; + (void) resume_fptr; assert(false); return true; } @@ -274,8 +273,8 @@ class IntMemoryStore : public DataStoreHandler const std::function *yield_fptr = nullptr, const std::function *resume_fptr = nullptr) override { - (void)yield_fptr; - (void)resume_fptr; + (void) yield_fptr; + (void) resume_fptr; assert(false); return true; } diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 4111c05e..a2fdedc1 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -178,9 +178,8 @@ LocalCcShards::LocalCcShards( timer_thd_ = std::thread([this] { TimerRun(); }); pthread_setname_np(timer_thd_.native_handle(), "tx_timer"); - // Start background thread to initialize table_ranges_heap. For mariadb, the - // main thread is the mariadb process thread; offload heap init to avoid - // blocking it. + // Start background thread to initialize table_ranges_heap. offload heap + // init to avoid concurrent access to heap with main thread. table_ranges_heap_thd_ = std::thread([this] { TableRangesHeapThreadRun(); }); pthread_setname_np(table_ranges_heap_thd_.native_handle(), @@ -2383,16 +2382,12 @@ std::pair LocalCcShards::PinStoreRange( ->BucketOwner() == ng_id; }()); - DLOG(INFO) << "fetch range slices, ng id = " << ng_id - << ", ng term = " << ng_term - << ", range table name = " << range_table_name.StringView(); // Fetch range slices info from data store if it's not loaded yet. range_entry->FetchRangeSlices( range_table_name, cc_request, ng_id, ng_term, cc_shard); return {range_entry, nullptr}; } - DLOG(INFO) << "store range slice count = " << store_range->SlicesCount(); return {range_entry, store_range}; } @@ -4858,9 +4853,6 @@ void LocalCcShards::DataSyncForHashPartition( data_sync_task->flight_task_cnt_ += 1; } - auto start_scan_time = std::chrono::steady_clock::now(); - size_t debug_data_size = 0; - for (size_t i = 0; i < partition_number_this_core; i += partition_number_per_scan) { @@ -5144,8 +5136,6 @@ void LocalCcShards::DataSyncForHashPartition( } #endif - debug_data_size += flush_data_size; - uint64_t old_usage = data_sync_mem_controller_.AllocateFlushDataMemQuota( flush_data_size); @@ -5346,12 +5336,6 @@ void LocalCcShards::DataSyncForHashPartition( } } - auto stop_scan_time = std::chrono::steady_clock::now(); - auto duration = std::chrono::duration_cast( - stop_scan_time - start_scan_time); - LOG(INFO) << "FlushDataImpl: scan duration = " << duration.count() << "us" - << ", data size = " << debug_data_size; - PostProcessHashPartitionDataSyncTask(std::move(data_sync_task), data_sync_txm, DataSyncTask::CkptErrorCode::NO_ERROR); @@ -5863,8 +5847,6 @@ void LocalCcShards::AddFlushTaskEntry(std::unique_ptr &&entry) auto &last_task = pending_flush_work.back(); if (last_task->MergeFrom(std::move(flush_data_task))) { - // LOG(INFO) << "AddFlushTaskEntry: Merge successful, task was " - // "merged into last_task"; // Merge successful, task was merged into last_task flush_data_worker_ctx_.cv_.notify_all(); return; @@ -5910,10 +5892,6 @@ void LocalCcShards::FlushCurrentFlushBuffer() auto &last_task = pending_flush_work.back(); if (last_task->MergeFrom(std::move(flush_data_task))) { - // LOG(INFO) << "FlushCurrentFlushBuffer: Merge successful, - // " - // "task was " - // "merged into last_task"; // Merge successful, task was merged into last_task flush_data_worker_ctx_.cv_.notify_all(); continue; @@ -5939,7 +5917,6 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, const std::function &yield_fn, const std::function &resume_fn) { - auto start_time = std::chrono::steady_clock::now(); auto &flush_task_entries = cur_work->flush_task_entries_; bool succ = true; @@ -6011,9 +5988,6 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, } } - LOG(INFO) << "FlushDataImpl: start update cce ts, task addr = " << cur_work - << ", worker idx = " << worker_idx; - std::unordered_set updated_ckpt_ts_core_ids; // Update cce ckpt ts in memory if (succ) @@ -6030,8 +6004,6 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, continue; } - auto start_time = std::chrono::steady_clock::now(); - absl::flat_hash_map> cce_entries_map; @@ -6063,14 +6035,6 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, } } - auto stop_time = std::chrono::steady_clock::now(); - auto duration = - std::chrono::duration_cast( - stop_time - start_time); - // LOG(INFO) << "FlushDataImpl: UpdateCceCkptTsCc duration = " - // << duration.count() << "us" - // << ", rec size = " << entry->data_sync_vec_->size(); - if (cce_entries_map.size() > 0) { UpdateCceCkptTsCc update_cce_req( @@ -6092,11 +6056,6 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, if (iterations_since_yield >= MAX_ITERATIONS_WITHOUT_YIELD) { - // LOG(INFO) - // << "FlushDataImpl: Before sync yield, task " - // "addr = " - // << cur_work << ", worker idx = " << - // worker_idx; sync_yield_func(); iterations_since_yield = 0; } @@ -6108,8 +6067,6 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, } } - // LOG(INFO) << "FlushDataImpl: start reset cc, task addr = " << cur_work - // << ", worker idx = " << worker_idx; // Notify cc shards that dirty data has been flushed. This will re-enqueue // kickout data cc reqs if there are any. WaitableCc reset_cc( @@ -6139,10 +6096,6 @@ void LocalCcShards::FlushDataImpl(FlushDataTask *cur_work, PostProcessFlushTaskEntries( flush_task_entries, ckpt_err, &yield_fn, &resume_fn, &sync_yield_func); - auto stop_time = std::chrono::steady_clock::now(); - auto duration = std::chrono::duration_cast( - stop_time - start_time); - LOG(INFO) << "FlushDataImpl duration = " << duration.count() << "us"; } void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk, @@ -6161,8 +6114,6 @@ void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk, flush_worker_lk.unlock(); - // Phase 2: Pass no-op callbacks; Phase 4/5 will pass real coroutine - // callbacks. FlushDataImpl(cur_work.get(), worker_idx, []() {}, []() {}, []() {}); flush_worker_lk.lock(); @@ -6189,8 +6140,6 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) { if (!pending_flush_work.empty()) { - // LOG(INFO) << "FlushDataWorker: flush work size = " - // << pending_flush_work.size(); std::unique_ptr cur_work = std::move(pending_flush_work.front()); pending_flush_work.pop_front(); @@ -6204,11 +6153,7 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) flush_coro_stack_allocator_, [this, ctx, worker_idx](continuation &&sink) { - ctx->yield_fn = [&sink]() - { - // LOG(INFO) << "CoroCtx yield_fn"; - sink = sink.resume(); - }; + ctx->yield_fn = [&sink]() { sink = sink.resume(); }; ctx->sync_yield_func = [&sink, weak_ctx = std::weak_ptr(ctx), @@ -6220,8 +6165,6 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) { std::lock_guard lk( flush_data_worker_ctx_.mux_); - // LOG(INFO) << "CoroCtx sync_yield_func: push " - // "ctx to resume_queue"; resume_queue_[worker_idx].push_back( std::move(c)); flush_data_worker_ctx_.cv_.notify_all(); @@ -6235,9 +6178,6 @@ void LocalCcShards::FlushDataWorker(size_t worker_idx) { if (auto c = weak_ctx.lock()) { - // LOG(INFO) - // << "CoroCtx resume_fn: coro ctx = " << - // c.get(); std::lock_guard lk( flush_data_worker_ctx_.mux_); resume_queue_[worker_idx].push_back(std::move(c)); diff --git a/tx_service/src/fault/log_replay_service.cpp b/tx_service/src/fault/log_replay_service.cpp index 40b91123..81335512 100644 --- a/tx_service/src/fault/log_replay_service.cpp +++ b/tx_service/src/fault/log_replay_service.cpp @@ -882,7 +882,7 @@ void RecoveryService::on_closed(brpc::StreamId id) } info = it->second.get(); } - DLOG(INFO) << "replay service stream: " << id << " wait and clear requests"; + WaitAndClearRequests(id, info->mux_, info->on_fly_cnt_, @@ -908,8 +908,6 @@ void RecoveryService::WaitAndClearRequests(brpc::StreamId stream_id, WaitingStatus waiting_status) { size_t on_fly_cnt = on_fly_cnt_.load(std::memory_order_relaxed); - DLOG(INFO) << "wait replay requests, stream id = " << stream_id - << ", on fly cnt = " << on_fly_cnt; if ((on_fly_cnt > 0 && waiting_status == WaitingStatus::WaitForAll) || (on_fly_cnt > 10000 && waiting_status == WaitingStatus::WaitForMany)) { @@ -925,7 +923,6 @@ void RecoveryService::WaitAndClearRequests(brpc::StreamId stream_id, } status.store(WaitingStatus::Active, std::memory_order_relaxed); } - DLOG(INFO) << "wait finished, stream id = " << stream_id; std::unique_lock lk(mux); if (recovery_error) { @@ -954,8 +951,6 @@ void RecoveryService::WaitAndClearRequests(brpc::StreamId stream_id, if (info->cc_ng_id_ == error_node_group_id && info->cc_ng_term_ == error_term) { - DLOG(INFO) << "RecoveryService, close stream, stream id = " - << stream_id; brpc::StreamClose(stream_id); } } diff --git a/tx_service/src/tx_index_operation.cpp b/tx_service/src/tx_index_operation.cpp index cebb4a87..1c852020 100644 --- a/tx_service/src/tx_index_operation.cpp +++ b/tx_service/src/tx_index_operation.cpp @@ -472,20 +472,26 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) break; } } - LOG(INFO) - << "Alter Table Index transaction unlock cluster config " - "lock failed, txn: " - << txm->TxNumber() << ", error code: " - << (int) unlock_cluster_config_op_.hd_result_.ErrorCode() - << ", error message: " - << unlock_cluster_config_op_.hd_result_.ErrorMsg() - << ", cluster config addr term" - << cluster_config_addr->Term() - << ", cluster config addr node group id: " - << cluster_config_addr->NodeGroupId() - << ", node group term: " - << Sharder::Instance().LeaderTerm( - cluster_config_addr->NodeGroupId()); + + if (cluster_config_addr) + { + DLOG(INFO) + << "Alter Table Index transaction unlock cluster " + "config " + "lock failed, txn: " + << txm->TxNumber() << ", error code: " + << (int) + unlock_cluster_config_op_.hd_result_.ErrorCode() + << ", error message: " + << unlock_cluster_config_op_.hd_result_.ErrorMsg() + << ", cluster config addr term" + << cluster_config_addr->Term() + << ", cluster config addr node group id: " + << cluster_config_addr->NodeGroupId() + << ", node group term: " + << Sharder::Instance().LeaderTerm( + cluster_config_addr->NodeGroupId()); + } // Releasing a local read lock should never fail assert(false); }