From 07c38af8ef069af883ee397d50fc5bf288310ea9 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Mon, 3 Aug 2026 09:16:06 +0000 Subject: [PATCH] feat: support ignoring Redis TTL for diagnostics --- data_substrate | 2 +- docs/03-data-model.md | 1 + eloqkv.ini | 7 ++++++ include/redis_hash_object.h | 2 +- include/redis_list_object.h | 2 +- include/redis_object.h | 20 +++++++++++++++++ include/redis_set_object.h | 2 +- include/redis_string_object.h | 2 +- include/redis_zset_object.h | 2 +- src/redis_service.cpp | 9 ++++++++ .../object_serialize_deserialize_test.cpp | 22 +++++++++++++++++++ 11 files changed, 65 insertions(+), 6 deletions(-) diff --git a/data_substrate b/data_substrate index 63db0ecb..99604109 160000 --- a/data_substrate +++ b/data_substrate @@ -1 +1 @@ -Subproject commit 63db0ecbe3cd04e23282aa61af3961491647bddf +Subproject commit 99604109c00967d33c07cc6adf167df51bd33331 diff --git a/docs/03-data-model.md b/docs/03-data-model.md index 4fc3fb3a..245f13d2 100644 --- a/docs/03-data-model.md +++ b/docs/03-data-model.md @@ -85,6 +85,7 @@ Key predicates as EloqKV implements them: - **Lazy enforcement (engine side)**: during ApplyCc the `ObjectCcMap` reads the payload's TTL; if expired, read-only commands return `RecordStatus::Deleted` immediately, and mutating commands set `ttl_expired_` so the engine first applies `RetireExpiredTTLObjectCommand()` (= `DelCommand`) and then runs the new command on a fresh object (`data_substrate/tx_service/include/cc/object_cc_map.h:657-673`, `:928`). Reads from the KV store likewise treat expired records as deleted, and physical reclamation is compaction-driven (`TTLCompactionFilter`) โ€” `data_substrate/docs/09-store-handler.md` ยง"Record TTL". In-memory pages track `smallest_ttl_` so page cleaning can drop expired entries wholesale (`cc/cc_page_clean_guard.h:78-123`). - **WAL correctness for TTL-only writes**: EXPIRE/PERSIST/GETEX mutate only the TTL, are *not* overwrites, yet replay must not depend on fetching the old value from the KV store. So when they will actually reset a TTL, `ExecuteOn` serializes the **whole current object** into an attached `RecoverObjectCommand` (`recover_ttl_obj_cmd_`, `src/redis_command.cpp:7923-7927`, `:7741-7742`); the engine logs that command's image instead (`RecoverTTLObjectCommand()` hook, `object_cc_map.h:1085`). `RecoverObjectCommand` is itself an overwrite whose `CommitOn` rebuilds the TTL object from the embedded blob (`src/redis_command.cpp:7601-7677`). For a key owned by a **remote** node group the coordinator (not the owner) writes the WAL, so the owner returns these facts in `ApplyResponse` (engine fields 10-13, eloqdata/eloqkv#509): `ttl_reset` + `recover_cmd_image` (the owner-serialized snapshot, logged as an overwrite record), the **post-command** object `ttl` (validity horizon, else the remote path logged `UINT64_MAX` and recovery could resurrect an expired object), and `ttl_expired` (the coordinator prepends the retire `DelCommand` record for the expired-then-recreated key). The owner always reports the post-command ttl โ€” an expired-recreation write reports `UINT64_MAX`, not the stale expired value, which also fixed a pre-existing *local* recovery loss (acknowledged expired-key recreations were discarded at replay). Old owners whose response omits fields 10-13 degrade to the previous behavior. - The checkpointer hands each record's TTL to the store handler (`BatchWriteRecords` items carry a ttl), keyed off `HasTTL()/GetTTL()` โ€” `object_cc_map.h:640-652`, engine doc 09. +- **Diagnostic TTL bypass**: RocksDB-backed EloqDSS builds accept `--ignore_redis_ttl=true` (or `[store] ignore_redis_ttl=true`). The flag makes Redis TTL objects report no TTL at runtime, bypasses the EloqDSS point-read/scan expiry filters, and disables physical TTL removal by the RocksDB compaction filter. It does **not** rewrite either the outer DSS TTL header or the TTL embedded in the Redis object blob; after all EloqKV and DSS processes restart with the flag disabled, normal expiration resumes from the original absolute timestamp. The mode is intended for isolated diagnostic clusters; ordinary writes still have their normal persistent effects. ## 7. The Catalog Factory (`include/eloqkv_catalog_factory.h`, `src/eloqkv_catalog_factory.cpp`) diff --git a/eloqkv.ini b/eloqkv.ini index 8c634cfa..9e495cad 100644 --- a/eloqkv.ini +++ b/eloqkv.ini @@ -98,6 +98,13 @@ node_group_replica_num = 1 # auto_redirect = false [store] +# Diagnostic mode for RocksDB/RocksDB Cloud only. When true, persisted Redis +# keys remain visible after their TTL expires and TTL/PTTL report no expiry. +# The stored TTL bytes are not changed; setting this back to false and +# restarting restores normal expiration. Enable it on every EloqKV and DSS +# process that accesses the diagnostic store. +# ignore_redis_ttl=false + # Shard data directories. # Format: path1,path2,...[,pathN][:weight1,weight2,...,weightN] # Weights are optional. When omitted, disk-capacity-based weighting is used. diff --git a/include/redis_hash_object.h b/include/redis_hash_object.h index 304b7eb1..6ef4adcf 100644 --- a/include/redis_hash_object.h +++ b/include/redis_hash_object.h @@ -267,7 +267,7 @@ class RedisHashTTLObject : public RedisHashObject bool HasTTL() const override { - return true; + return !IgnoreTTL(); } RedisObjectType ObjectType() const override diff --git a/include/redis_list_object.h b/include/redis_list_object.h index a622b405..f18dbe4f 100644 --- a/include/redis_list_object.h +++ b/include/redis_list_object.h @@ -263,7 +263,7 @@ struct RedisListTTLObject : public RedisListObject bool HasTTL() const override { - return true; + return !IgnoreTTL(); } RedisObjectType ObjectType() const override diff --git a/include/redis_object.h b/include/redis_object.h index 0a2242fe..9c6f03d7 100644 --- a/include/redis_object.h +++ b/include/redis_object.h @@ -22,6 +22,7 @@ #pragma once #include +#include #include #include #include @@ -62,6 +63,22 @@ enum struct RedisObjectType struct RedisEloqObject : public txservice::TxObject { public: + /** + * Controls the diagnostic mode that exposes persisted Redis objects even + * after their expiration timestamp. The mode changes only runtime TTL + * interpretation; serialized TTL metadata remains intact so disabling the + * mode on a later restart restores normal expiration. + */ + static void SetIgnoreTTL(bool ignore_ttl) + { + ignore_ttl_.store(ignore_ttl, std::memory_order_release); + } + + static bool IgnoreTTL() + { + return ignore_ttl_.load(std::memory_order_acquire); + } + TxRecord::Uptr Clone() const override { assert(false); @@ -128,5 +145,8 @@ struct RedisEloqObject : public txservice::TxObject { return std::make_unique(); } + +private: + inline static std::atomic_bool ignore_ttl_{false}; }; } // namespace EloqKV diff --git a/include/redis_set_object.h b/include/redis_set_object.h index 4f302f9b..b0b571cd 100644 --- a/include/redis_set_object.h +++ b/include/redis_set_object.h @@ -131,7 +131,7 @@ class RedisHashSetTTLObject : public RedisHashSetObject bool HasTTL() const override { - return true; + return !IgnoreTTL(); } RedisObjectType ObjectType() const override diff --git a/include/redis_string_object.h b/include/redis_string_object.h index 06085fd4..7f698ac9 100644 --- a/include/redis_string_object.h +++ b/include/redis_string_object.h @@ -282,7 +282,7 @@ struct RedisStringTTLObject : public RedisStringObject bool HasTTL() const override { - return true; + return !IgnoreTTL(); } RedisObjectType ObjectType() const override diff --git a/include/redis_zset_object.h b/include/redis_zset_object.h index 813bee2a..bdba96d8 100644 --- a/include/redis_zset_object.h +++ b/include/redis_zset_object.h @@ -375,7 +375,7 @@ class RedisZsetTTLObject : public RedisZsetObject bool HasTTL() const override { - return true; + return !IgnoreTTL(); } RedisObjectType ObjectType() const override diff --git a/src/redis_service.cpp b/src/redis_service.cpp index 05c1bb88..2648c27e 100644 --- a/src/redis_service.cpp +++ b/src/redis_service.cpp @@ -58,6 +58,7 @@ #include "b255.h" #include "catalog_factory.h" #include "data_substrate.h" +#include "eloq_data_store_service/ignore_redis_ttl.h" #include "eloq_metrics/include/metrics.h" #include "eloqkv_key.h" #include "error_messages.h" @@ -240,6 +241,14 @@ bool RedisServiceImpl::Init(brpc::Server &brpc_server) return false; } + const bool ignore_redis_ttl = EloqDS::IgnoreRedisTTL(); + RedisEloqObject::SetIgnoreTTL(ignore_redis_ttl); + if (ignore_redis_ttl) + { + LOG(WARNING) << "ignore_redis_ttl is enabled: persisted expiration " + "timestamps will be retained but not enforced"; + } + // Engine registration: EloqKv auto &ds = DataSubstrate::Instance(); diff --git a/tests/unit/eloq/object_serialize_deserialize_test.cpp b/tests/unit/eloq/object_serialize_deserialize_test.cpp index f2a25614..0ec8a2d0 100644 --- a/tests/unit/eloq/object_serialize_deserialize_test.cpp +++ b/tests/unit/eloq/object_serialize_deserialize_test.cpp @@ -12,6 +12,7 @@ #include "redis_errors.h" #include "redis_hash_object.h" #include "redis_list_object.h" +#include "redis_string_object.h" #include "redis_zset_object.h" absl::flat_hash_map EloqKV::RedisZsetObject::* @@ -28,6 +29,27 @@ struct Rob } }; +TEST_CASE("ignore Redis TTL preserves serialized metadata") +{ + EloqKV::RedisStringTTLObject object; + object.SetTTL(12345); + + EloqKV::RedisEloqObject::SetIgnoreTTL(false); + REQUIRE(object.HasTTL()); + std::string before; + object.Serialize(before); + + EloqKV::RedisEloqObject::SetIgnoreTTL(true); + REQUIRE_FALSE(object.HasTTL()); + REQUIRE(object.GetTTL() == 12345); + std::string ignored; + object.Serialize(ignored); + REQUIRE(ignored == before); + + EloqKV::RedisEloqObject::SetIgnoreTTL(false); + REQUIRE(object.HasTTL()); +} + TEST_CASE("zset_object-string") { LOG(INFO) << "running: zset_object-string: ";